feat: add compression negotiation for sent messages (#91)

feat: add compression negotiation for sent messages
fix: unix homedir handling

the service will negotiate a compression algo for sending messages
when a user chats someone during the auto discovery, the service returns an `Accept-Encoding: br, gzip, deflate`

the client saves that response and so when it makes POSTs of messages adds the best `Content-Encoding` and compresses the message

example:
```
>> GET /.well-known/salty/c765c69040d98f3af2181237f47ec01398d80f8ab2690fe929e4311ab05dec01.json

<< Accept-Encoding: br, gzip, deflate
<<
<< {"endpoint":"https://salty.home.arpa/inbox/01FZBR8Y2E6TH949JA3925WF71","key":"kex1wurry09ftqjuxgjl0jxmqypv4axqvzqljkgeadxjcpwtfuhcedcslck52d"}

>> POST /inbox/01FZBR8Y2E6TH949JA3925WF71
>> Content-Encoding: br
>>
>> [Brotli Compressed data]
```

this PR depends on prologic/msgbus#24

Co-authored-by: Jon Lundy <jon@xuu.cc>
Reviewed-on: #91
Co-authored-by: xuu <xuu@noreply@mills.io>
Co-committed-by: xuu <xuu@noreply@mills.io>

Caddyfile

@@ -1,9 +1,5 @@
 salty.home.arpa {
   tls internal
 
-  file_server * browse {
-      root data/
-  }
-
-  reverse_proxy /inbox/* http://127.0.0.1:8000
+  reverse_proxy http://127.0.0.1:8000
 }

client.go

@@ -15,6 +15,7 @@ import (
 	"github.com/avast/retry-go"
 	"github.com/keys-pub/keys"
 	log "github.com/sirupsen/logrus"
+	"github.com/timewasted/go-accept-headers"
 
 	"go.mills.io/salty"
 	"go.mills.io/saltyim/internal/exec"
@@ -28,6 +29,8 @@ const (
 var (
 	ErrNoMessages         = errors.New("error: no messages found")
 	ErrClientNotConnected = errors.New("error: client not connected")
+
+	acceptEncodings = []string{"br", "gzip", ""}
 )
 
 type addrCache map[string]*Addr
@@ -66,8 +69,18 @@ func PackMessage(me *Addr, msg string) []byte {
 
 // Send sends the encrypted message `msg` to the Endpoint `endpoint` using a
 // `POST` request and returns nil on success or an error on failure.
-func Send(endpoint, msg string) error {
-	res, err := Request(http.MethodPost, endpoint, nil, bytes.NewBufferString(msg))
+func Send(endpoint, msg string, cap Capabilities) error {
+	headers := make(http.Header)
+
+	if cap.AcceptEncoding != "" {
+		ae, err := accept.Negotiate(cap.AcceptEncoding, acceptEncodings...)
+		if err != nil {
+			return fmt.Errorf("error publishing message to %s: %w", endpoint, err)
+		}
+		headers.Set("Content-Encoding", ae)
+	}
+
+	res, err := Request(http.MethodPost, endpoint, headers, bytes.NewBufferString(msg))
 	if err != nil {
 		return fmt.Errorf("error publishing message to %s: %w", endpoint, err)
 	}
@@ -329,7 +342,7 @@ func (cli *Client) SendToAddr(addr *Addr, msg string) error {
 		return fmt.Errorf("error encrypting message to %s: %w", addr, err)
 	}
 
-	if err := Send(addr.Endpoint().String(), string(b)); err != nil {
+	if err := Send(addr.Endpoint().String(), string(b), addr.Cap()); err != nil {
 		return fmt.Errorf("error sending message to %s: %w", addr, err)
 	}
 

cmd/salty-chat/makeuser.go

@@ -102,6 +102,8 @@ func makeuser(me *saltyim.Addr, identity, endpoint string) {
 	}
 
 	dir := filepath.Dir(identity)
+	dir = saltyim.FixUnixHome(dir)
+
 	if err := os.MkdirAll(dir, 0700); err != nil {
 		fmt.Fprintf(os.Stderr, "error creating configuration directory %s: %s\n", dir, err)
 		os.Exit(2)

cmd/salty-chat/register.go

@@ -42,6 +42,8 @@ func init() {
 
 func register(me *saltyim.Addr, identity string) {
 	dir := filepath.Dir(identity)
+	dir = saltyim.FixUnixHome(dir)
+
 	if err := os.MkdirAll(dir, 0700); err != nil {
 		fmt.Fprintf(os.Stderr, "error creating configuration directory %s: %s\n", dir, err)
 		os.Exit(2)

go.mod

@@ -2,11 +2,10 @@ module go.mills.io/saltyim
 
 go 1.17
 
-//replace github.com/mlctrez/goapp-mdc v0.2.5 => ../goapp-mdc
-
-//replace git.mills.io/prologic/msgbus => ../../msgbus
+//replace git.mills.io/prologic/msgbus => ../msgbus
 
 require (
+	github.com/andybalholm/brotli v1.0.4
 	github.com/avast/retry-go v2.7.0+incompatible
 	github.com/likexian/doh-go v0.6.4
 	github.com/mitchellh/go-homedir v1.1.0
@@ -17,6 +16,7 @@ require (
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.10.1
 	github.com/stretchr/testify v1.7.0
+	github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09
 	go.mills.io/salty v0.0.0-20220322161301-ce2b9f6573fa
 	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
@@ -80,7 +80,7 @@ require (
 
 require (
 	git.mills.io/prologic/bitcask v1.0.2
-	git.mills.io/prologic/msgbus v0.1.13-0.20220329013040-e6ab9d9c70c1
+	git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6
 	git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1
 	git.mills.io/prologic/useragent v0.0.0-20210714100044-d249fe7921a0
 	github.com/NYTimes/gziphandler v1.1.1

go.sum

@@ -51,8 +51,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 git.mills.io/prologic/bitcask v1.0.2 h1:Iy9x3mVVd1fB+SWY0LTmsSDPGbzMrd7zCZPKbsb/tDA=
 git.mills.io/prologic/bitcask v1.0.2/go.mod h1:ppXpR3haeYrijyJDleAkSGH3p90w6sIHxEA/7UHMxH4=
-git.mills.io/prologic/msgbus v0.1.13-0.20220329013040-e6ab9d9c70c1 h1:CCgyDRt8tDf2OOoEHHtK9aBsl00U4RyXGlrdBPTof+g=
-git.mills.io/prologic/msgbus v0.1.13-0.20220329013040-e6ab9d9c70c1/go.mod h1:2YmGBm9WJjfMTBki/PuD5eG0CUULXesaV6kpVF/jJ2g=
+git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6 h1:9Ci4a+yqtRdnj8JitXaRGntxeAkdFe+NltnR2ehl4vo=
+git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6/go.mod h1:UyiNBmWbpsq7mtO+FHWoGwRiccPcT+EJGqT/idm/lfo=
 git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1 h1:e6ZyAOFGLZJZYL2galNvfuNMqeQDdilmQ5WRBXCNL5s=
 git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1/go.mod h1:/rNXqsTHGrilgNJYH/8wsIRDScyxXUhpbSdNbBatAKY=
 git.mills.io/prologic/useragent v0.0.0-20210714100044-d249fe7921a0 h1:MojWEgZyiugUbgyjydrdSAkHlADnbt90dXyURRYFzQ4=
@@ -72,6 +72,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -575,6 +577,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
 github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/redcon v1.4.1/go.mod h1:XwNPFbJ4ShWNNSA2Jazhbdje6jegTCcwFR6mfaADvHA=
+github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09 h1:QVxbx5l/0pzciWYOynixQMtUhPYC3YKD6EcUlOsgGqw=
+github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09/go.mod h1:Uy/Rnv5WKuOO+PuDhuYLEpUiiKIZtss3z519uk67aF0=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=

identity.go

@@ -62,6 +62,7 @@ func CreateIdentity(options ...IdentityOption) (*Identity, error) {
 	}
 
 	fn := ident.path
+	fn = FixUnixHome(fn)
 
 	f, err := os.OpenFile(fn, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
 	if err != nil {
@@ -97,11 +98,7 @@ func GetIdentity(options ...IdentityOption) (*Identity, error) {
 
 	if ident.path != "" {
 		fn := ident.path
-		// Handle unix home with `~`
-		if strings.HasPrefix(fn, "~/") {
-			dirname, _ := os.UserHomeDir()
-			fn = filepath.Join(dirname, fn[2:])
-		}
+		fn = FixUnixHome(fn)
 
 		id, err := os.Open(fn)
 		if err != nil {
@@ -195,3 +192,13 @@ func WithIdentityPath(path string) IdentityOption {
 func WithIdentityBytes(contents []byte) IdentityOption {
 	return func(i *Identity) { i.contents = contents }
 }
+
+// Handle unix home with `~`
+func FixUnixHome(p string) string {
+	// Handle unix home with `~`
+	if strings.HasPrefix(p, "~/") {
+		dirname, _ := os.UserHomeDir()
+		return filepath.Join(dirname, p[2:])
+	}
+	return p
+}

internal/handlers.go

@@ -23,6 +23,7 @@ func (s *Server) NotFoundHandler(w http.ResponseWriter, r *http.Request) {
 func (s *Server) ConfigHandler() httprouter.Handle {
 	configDir := filepath.Join(s.config.Data, wellknownPath)
 	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		w.Header().Set("Accept-Encoding", "br, gzip, deflate")
 		http.ServeFile(w, r, filepath.Join(configDir, p.ByName("config")))
 	}
 }

internal/tasks.go

@@ -15,6 +15,8 @@ const (
 
 func CreateConfig(conf *Config, hash string, key string) error {
 	p := filepath.Join(conf.Data, wellknownPath)
+	p = saltyim.FixUnixHome(p)
+
 	fn := filepath.Join(p, fmt.Sprintf("%s.json", hash))
 	if FileExists(fn) {
 		return nil

lookup.go

@@ -16,19 +16,23 @@ var (
 	_ json.Marshaler = (*Addr)(nil)
 )
 
-func fetchConfig(addr string) (Config, error) {
+func fetchConfig(addr string) (Config, Capabilities, error) {
 	// Attempt using hash
 	res, err := Request(http.MethodGet, addr, nil, nil)
 	if err != nil {
-		return Config{}, err
+		return Config{}, Capabilities{}, err
 	}
 
 	var config Config
 	if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
-		return Config{}, err
+		return Config{}, Capabilities{}, err
 	}
 
-	return config, err
+	cap := Capabilities{
+		AcceptEncoding: res.Header.Get("Accept-Encoding"),
+	}
+
+	return config, cap, err
 }
 
 // Config represents a Salty Config for a User which at a minimum is required
@@ -38,6 +42,10 @@ type Config struct {
 	Key      string `json:"key"`
 }
 
+type Capabilities struct {
+	AcceptEncoding string
+}
+
 // Addr represents a Salty IM User's Address
 type Addr struct {
 	User   string
@@ -46,6 +54,7 @@ type Addr struct {
 	key              *keys.EdX25519PublicKey
 	endpoint         *url.URL
 	discoveredDomain string
+	capabilities     Capabilities
 }
 
 // IsZero returns true if the address is empty
@@ -92,7 +101,11 @@ func (a *Addr) Key() *keys.EdX25519PublicKey {
 // Endpoint returns the discovered Endpoint
 func (a *Addr) Endpoint() *url.URL {
 	return a.endpoint
+}
 
+// Cap returns the discovered Capabilities
+func (a *Addr) Cap() Capabilities {
+	return a.capabilities
 }
 
 // DiscoveredDomain returns the discovered domain (if any) of fallbacks to the Domain
@@ -122,10 +135,10 @@ func (a *Addr) Refresh() error {
 		log.Debugf("error looking up SRV record for _salty._tcp.%s : %s", a.Domain, err)
 	}
 
-	config, err := fetchConfig(a.HashURI())
+	config, cap, err := fetchConfig(a.HashURI())
 	if err != nil {
 		// Fallback to plain user nick
-		config, err = fetchConfig(a.URI())
+		config, cap, err = fetchConfig(a.URI())
 	}
 	if err != nil {
 		return fmt.Errorf("error looking up user %s: %w", a, err)
@@ -141,6 +154,7 @@ func (a *Addr) Refresh() error {
 		return fmt.Errorf("error parsing endpoint %s: %w", config.Endpoint, err)
 	}
 	a.endpoint = u
+	a.capabilities = cap
 
 	return nil
 }

utils.go

@@ -1,6 +1,8 @@
 package saltyim
 
 import (
+	"bytes"
+	"compress/gzip"
 	"crypto/rand"
 	"fmt"
 	"io"
@@ -8,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/andybalholm/brotli"
 	"github.com/oklog/ulid/v2"
 	log "github.com/sirupsen/logrus"
 )
@@ -40,15 +43,32 @@ func MustGenerateULID() string {
 // requests to Salty endpoints for looking up Salty Addresses, Configs and
 // publishing encrypted messages.
 func Request(method, uri string, headers http.Header, body io.Reader) (*http.Response, error) {
+	if headers == nil {
+		headers = make(http.Header)
+	}
+
+	if body != nil {
+		switch headers.Get("Content-Encoding") {
+		case "br":
+			buf := &bytes.Buffer{}
+			br := brotli.NewWriter(buf)
+			io.Copy(br, body)
+			br.Close()
+			body = buf
+		case "gzip":
+			buf := &bytes.Buffer{}
+			gz := gzip.NewWriter(buf)
+			io.Copy(gz, body)
+			gz.Close()
+			body = buf
+		}
+	}
+
 	req, err := http.NewRequest(method, uri, body)
 	if err != nil {
 		return nil, fmt.Errorf("%s: http.NewRequest fail: %s", uri, err)
 	}
 
-	if headers == nil {
-		headers = make(http.Header)
-	}
-
 	// Set a default User-Agent (if none set)
 	if headers.Get("User-Agent") == "" {
 		headers.Set("User-Agent", fmt.Sprintf("saltyim/%s", FullVersion()))