feat: add compression negotiation for sent messages (#91)

feat: add compression negotiation for sent messages
fix: unix homedir handling

the service will negotiate a compression algo for sending messages
when a user chats someone during the auto discovery, the service returns an `Accept-Encoding: br, gzip, deflate`

the client saves that response and so when it makes POSTs of messages adds the best `Content-Encoding` and compresses the message

example:
```
>> GET /.well-known/salty/c765c69040d98f3af2181237f47ec01398d80f8ab2690fe929e4311ab05dec01.json

<< Accept-Encoding: br, gzip, deflate
<<
<< {"endpoint":"https://salty.home.arpa/inbox/01FZBR8Y2E6TH949JA3925WF71","key":"kex1wurry09ftqjuxgjl0jxmqypv4axqvzqljkgeadxjcpwtfuhcedcslck52d"}

>> POST /inbox/01FZBR8Y2E6TH949JA3925WF71
>> Content-Encoding: br
>>
>> [Brotli Compressed data]
```

this PR depends on prologic/msgbus#24

Co-authored-by: Jon Lundy <jon@xuu.cc>
Reviewed-on: #91
Co-authored-by: xuu <xuu@noreply@mills.io>
Co-committed-by: xuu <xuu@noreply@mills.io>
pull/92/head
xuu 6 months ago committed by James Mills
parent f9fa628e75
commit 754fcc7323
  1. 6
      Caddyfile
  2. 19
      client.go
  3. 2
      cmd/salty-chat/makeuser.go
  4. 2
      cmd/salty-chat/register.go
  5. 8
      go.mod
  6. 8
      go.sum
  7. 17
      identity.go
  8. 1
      internal/handlers.go
  9. 2
      internal/tasks.go
  10. BIN
      internal/web/app.wasm
  11. 26
      lookup.go
  12. 28
      utils.go

@ -1,9 +1,5 @@
salty.home.arpa {
tls internal
file_server * browse {
root data/
}
reverse_proxy /inbox/* http://127.0.0.1:8000
reverse_proxy http://127.0.0.1:8000
}

@ -15,6 +15,7 @@ import (
"github.com/avast/retry-go"
"github.com/keys-pub/keys"
log "github.com/sirupsen/logrus"
"github.com/timewasted/go-accept-headers"
"go.mills.io/salty"
"go.mills.io/saltyim/internal/exec"
@ -28,6 +29,8 @@ const (
var (
ErrNoMessages = errors.New("error: no messages found")
ErrClientNotConnected = errors.New("error: client not connected")
acceptEncodings = []string{"br", "gzip", ""}
)
type addrCache map[string]*Addr
@ -66,8 +69,18 @@ func PackMessage(me *Addr, msg string) []byte {
// Send sends the encrypted message `msg` to the Endpoint `endpoint` using a
// `POST` request and returns nil on success or an error on failure.
func Send(endpoint, msg string) error {
res, err := Request(http.MethodPost, endpoint, nil, bytes.NewBufferString(msg))
func Send(endpoint, msg string, cap Capabilities) error {
headers := make(http.Header)
if cap.AcceptEncoding != "" {
ae, err := accept.Negotiate(cap.AcceptEncoding, acceptEncodings...)
if err != nil {
return fmt.Errorf("error publishing message to %s: %w", endpoint, err)
}
headers.Set("Content-Encoding", ae)
}
res, err := Request(http.MethodPost, endpoint, headers, bytes.NewBufferString(msg))
if err != nil {
return fmt.Errorf("error publishing message to %s: %w", endpoint, err)
}
@ -329,7 +342,7 @@ func (cli *Client) SendToAddr(addr *Addr, msg string) error {
return fmt.Errorf("error encrypting message to %s: %w", addr, err)
}
if err := Send(addr.Endpoint().String(), string(b)); err != nil {
if err := Send(addr.Endpoint().String(), string(b), addr.Cap()); err != nil {
return fmt.Errorf("error sending message to %s: %w", addr, err)
}

@ -102,6 +102,8 @@ func makeuser(me *saltyim.Addr, identity, endpoint string) {
}
dir := filepath.Dir(identity)
dir = saltyim.FixUnixHome(dir)
if err := os.MkdirAll(dir, 0700); err != nil {
fmt.Fprintf(os.Stderr, "error creating configuration directory %s: %s\n", dir, err)
os.Exit(2)

@ -42,6 +42,8 @@ func init() {
func register(me *saltyim.Addr, identity string) {
dir := filepath.Dir(identity)
dir = saltyim.FixUnixHome(dir)
if err := os.MkdirAll(dir, 0700); err != nil {
fmt.Fprintf(os.Stderr, "error creating configuration directory %s: %s\n", dir, err)
os.Exit(2)

@ -2,11 +2,10 @@ module go.mills.io/saltyim
go 1.17
//replace github.com/mlctrez/goapp-mdc v0.2.5 => ../goapp-mdc
//replace git.mills.io/prologic/msgbus => ../../msgbus
//replace git.mills.io/prologic/msgbus => ../msgbus
require (
github.com/andybalholm/brotli v1.0.4
github.com/avast/retry-go v2.7.0+incompatible
github.com/likexian/doh-go v0.6.4
github.com/mitchellh/go-homedir v1.1.0
@ -17,6 +16,7 @@ require (
github.com/spf13/cobra v1.4.0
github.com/spf13/viper v1.10.1
github.com/stretchr/testify v1.7.0
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09
go.mills.io/salty v0.0.0-20220322161301-ce2b9f6573fa
golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
@ -80,7 +80,7 @@ require (
require (
git.mills.io/prologic/bitcask v1.0.2
git.mills.io/prologic/msgbus v0.1.13-0.20220329013040-e6ab9d9c70c1
git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6
git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1
git.mills.io/prologic/useragent v0.0.0-20210714100044-d249fe7921a0
github.com/NYTimes/gziphandler v1.1.1

@ -51,8 +51,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.mills.io/prologic/bitcask v1.0.2 h1:Iy9x3mVVd1fB+SWY0LTmsSDPGbzMrd7zCZPKbsb/tDA=
git.mills.io/prologic/bitcask v1.0.2/go.mod h1:ppXpR3haeYrijyJDleAkSGH3p90w6sIHxEA/7UHMxH4=
git.mills.io/prologic/msgbus v0.1.13-0.20220329013040-e6ab9d9c70c1 h1:CCgyDRt8tDf2OOoEHHtK9aBsl00U4RyXGlrdBPTof+g=
git.mills.io/prologic/msgbus v0.1.13-0.20220329013040-e6ab9d9c70c1/go.mod h1:2YmGBm9WJjfMTBki/PuD5eG0CUULXesaV6kpVF/jJ2g=
git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6 h1:9Ci4a+yqtRdnj8JitXaRGntxeAkdFe+NltnR2ehl4vo=
git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6/go.mod h1:UyiNBmWbpsq7mtO+FHWoGwRiccPcT+EJGqT/idm/lfo=
git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1 h1:e6ZyAOFGLZJZYL2galNvfuNMqeQDdilmQ5WRBXCNL5s=
git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1/go.mod h1:/rNXqsTHGrilgNJYH/8wsIRDScyxXUhpbSdNbBatAKY=
git.mills.io/prologic/useragent v0.0.0-20210714100044-d249fe7921a0 h1:MojWEgZyiugUbgyjydrdSAkHlADnbt90dXyURRYFzQ4=
@ -72,6 +72,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@ -575,6 +577,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/redcon v1.4.1/go.mod h1:XwNPFbJ4ShWNNSA2Jazhbdje6jegTCcwFR6mfaADvHA=
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09 h1:QVxbx5l/0pzciWYOynixQMtUhPYC3YKD6EcUlOsgGqw=
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09/go.mod h1:Uy/Rnv5WKuOO+PuDhuYLEpUiiKIZtss3z519uk67aF0=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=

@ -62,6 +62,7 @@ func CreateIdentity(options ...IdentityOption) (*Identity, error) {
}
fn := ident.path
fn = FixUnixHome(fn)
f, err := os.OpenFile(fn, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
@ -97,11 +98,7 @@ func GetIdentity(options ...IdentityOption) (*Identity, error) {
if ident.path != "" {
fn := ident.path
// Handle unix home with `~`
if strings.HasPrefix(fn, "~/") {
dirname, _ := os.UserHomeDir()
fn = filepath.Join(dirname, fn[2:])
}
fn = FixUnixHome(fn)
id, err := os.Open(fn)
if err != nil {
@ -195,3 +192,13 @@ func WithIdentityPath(path string) IdentityOption {
func WithIdentityBytes(contents []byte) IdentityOption {
return func(i *Identity) { i.contents = contents }
}
// Handle unix home with `~`
func FixUnixHome(p string) string {
// Handle unix home with `~`
if strings.HasPrefix(p, "~/") {
dirname, _ := os.UserHomeDir()
return filepath.Join(dirname, p[2:])
}
return p
}

@ -23,6 +23,7 @@ func (s *Server) NotFoundHandler(w http.ResponseWriter, r *http.Request) {
func (s *Server) ConfigHandler() httprouter.Handle {
configDir := filepath.Join(s.config.Data, wellknownPath)
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
w.Header().Set("Accept-Encoding", "br, gzip, deflate")
http.ServeFile(w, r, filepath.Join(configDir, p.ByName("config")))
}
}

@ -15,6 +15,8 @@ const (
func CreateConfig(conf *Config, hash string, key string) error {
p := filepath.Join(conf.Data, wellknownPath)
p = saltyim.FixUnixHome(p)
fn := filepath.Join(p, fmt.Sprintf("%s.json", hash))
if FileExists(fn) {
return nil

BIN
internal/web/app.wasm (Stored with Git LFS)

Binary file not shown.

@ -16,19 +16,23 @@ var (
_ json.Marshaler = (*Addr)(nil)
)
func fetchConfig(addr string) (Config, error) {
func fetchConfig(addr string) (Config, Capabilities, error) {
// Attempt using hash
res, err := Request(http.MethodGet, addr, nil, nil)
if err != nil {
return Config{}, err
return Config{}, Capabilities{}, err
}
var config Config
if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
return Config{}, err
return Config{}, Capabilities{}, err
}
return config, err
cap := Capabilities{
AcceptEncoding: res.Header.Get("Accept-Encoding"),
}
return config, cap, err
}
// Config represents a Salty Config for a User which at a minimum is required
@ -38,6 +42,10 @@ type Config struct {
Key string `json:"key"`
}
type Capabilities struct {
AcceptEncoding string
}
// Addr represents a Salty IM User's Address
type Addr struct {
User string
@ -46,6 +54,7 @@ type Addr struct {
key *keys.EdX25519PublicKey
endpoint *url.URL
discoveredDomain string
capabilities Capabilities
}
// IsZero returns true if the address is empty
@ -92,7 +101,11 @@ func (a *Addr) Key() *keys.EdX25519PublicKey {
// Endpoint returns the discovered Endpoint
func (a *Addr) Endpoint() *url.URL {
return a.endpoint
}
// Cap returns the discovered Capabilities
func (a *Addr) Cap() Capabilities {
return a.capabilities
}
// DiscoveredDomain returns the discovered domain (if any) of fallbacks to the Domain
@ -122,10 +135,10 @@ func (a *Addr) Refresh() error {
log.Debugf("error looking up SRV record for _salty._tcp.%s : %s", a.Domain, err)
}
config, err := fetchConfig(a.HashURI())
config, cap, err := fetchConfig(a.HashURI())
if err != nil {
// Fallback to plain user nick
config, err = fetchConfig(a.URI())
config, cap, err = fetchConfig(a.URI())
}
if err != nil {
return fmt.Errorf("error looking up user %s: %w", a, err)
@ -141,6 +154,7 @@ func (a *Addr) Refresh() error {
return fmt.Errorf("error parsing endpoint %s: %w", config.Endpoint, err)
}
a.endpoint = u
a.capabilities = cap
return nil
}

@ -1,6 +1,8 @@
package saltyim
import (
"bytes"
"compress/gzip"
"crypto/rand"
"fmt"
"io"
@ -8,6 +10,7 @@ import (
"strings"
"time"
"github.com/andybalholm/brotli"
"github.com/oklog/ulid/v2"
log "github.com/sirupsen/logrus"
)
@ -40,15 +43,32 @@ func MustGenerateULID() string {
// requests to Salty endpoints for looking up Salty Addresses, Configs and
// publishing encrypted messages.
func Request(method, uri string, headers http.Header, body io.Reader) (*http.Response, error) {
if headers == nil {
headers = make(http.Header)
}
if body != nil {
switch headers.Get("Content-Encoding") {
case "br":
buf := &bytes.Buffer{}
br := brotli.NewWriter(buf)
io.Copy(br, body)
br.Close()
body = buf
case "gzip":
buf := &bytes.Buffer{}
gz := gzip.NewWriter(buf)
io.Copy(gz, body)
gz.Close()
body = buf
}
}
req, err := http.NewRequest(method, uri, body)
if err != nil {
return nil, fmt.Errorf("%s: http.NewRequest fail: %s", uri, err)
}
if headers == nil {
headers = make(http.Header)
}
// Set a default User-Agent (if none set)
if headers.Get("User-Agent") == "" {
headers.Set("User-Agent", fmt.Sprintf("saltyim/%s", FullVersion()))

Loading…
Cancel
Save