Use the new and improved with write-ahead-log (wal) support (#136)

Improves how inbox indices are peristed, hopefully much better now as the topic sequences are now a proper monotonic increasing integer, messages survive crashes/resrarts and so forth.

- [] Remove the `go.mod` replace directive after prologic/msgbus#33 is merged

cc @xuu

Co-authored-by: James Mills <prologic@shortcircuit.net.au>
Reviewed-on: #136
Reviewed-by: xuu <xuu@noreply@mills.io>
pull/140/head
James Mills 6 months ago
parent 237dce3664
commit e8110e2a86
  1. 2
      .gitignore
  2. 7
      cmd/salty-chat/chat.go
  3. 6
      cmd/salty-chat/read.go
  4. 0
      data/logs/.gitkeep
  5. 11
      go.mod
  6. 28
      go.sum
  7. 8
      internal/handlers.go
  8. 15
      internal/server.go
  9. BIN
      internal/web/app.wasm
  10. 14
      state.go

2
.gitignore vendored

@ -18,6 +18,8 @@
/data/*.db
/data/*.key
/data/logs
/data/acme
/data/avatars
/data/.well-known

@ -5,6 +5,7 @@ import (
"os"
"strings"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@ -50,12 +51,18 @@ func init() {
func chat(me *saltyim.Addr, identity, state, user string) {
cli, err := saltyim.NewClient(me,
saltyim.WithStateFromFile(state),
saltyim.WithClientIdentity(saltyim.WithIdentityPath(identity)),
)
if err != nil {
fmt.Fprintf(os.Stderr, "error initializing client: %s\n", err)
os.Exit(2)
}
defer func() {
if err := cli.State().Save(state); err != nil {
log.WithError(err).Warnf("error saving state: %s", state)
}
}()
// Set terminal title
tui.SetTerminalTitle("Salty IM with %s", user)

@ -96,12 +96,18 @@ func init() {
func read(me *saltyim.Addr, identity, state string, follow bool, extraenvs, prehook, posthook string, args ...string) {
cli, err := saltyim.NewClient(me,
saltyim.WithStateFromFile(state),
saltyim.WithClientIdentity(saltyim.WithIdentityPath(identity)),
)
if err != nil {
fmt.Fprintf(os.Stderr, "error initializing client: %s\n", err)
os.Exit(2)
}
defer func() {
if err := cli.State().Save(state); err != nil {
log.WithError(err).Warnf("error saving state: %s", state)
}
}()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

@ -69,12 +69,19 @@ require (
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tidwall/gjson v1.14.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/tinylru v1.1.0 // indirect
github.com/tidwall/wal v1.1.7 // indirect
github.com/tyler-smith/go-bip39 v1.1.0 // indirect
github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/writeas/go-strip-markdown/v2 v2.1.1 // indirect
golang.org/x/exp v0.0.0-20220328175248-053ad81199eb // indirect
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f // indirect
golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.7 // indirect
@ -87,7 +94,7 @@ require (
require (
git.mills.io/prologic/bitcask v1.0.2
git.mills.io/prologic/msgbus v0.1.14
git.mills.io/prologic/msgbus v0.1.15-0.20220403155938-6bfb669347cc
git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1
git.mills.io/prologic/useragent v0.0.0-20210714100044-d249fe7921a0
github.com/NYTimes/gziphandler v1.1.1

@ -42,14 +42,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.mills.io/prologic/bitcask v1.0.2 h1:Iy9x3mVVd1fB+SWY0LTmsSDPGbzMrd7zCZPKbsb/tDA=
git.mills.io/prologic/bitcask v1.0.2/go.mod h1:ppXpR3haeYrijyJDleAkSGH3p90w6sIHxEA/7UHMxH4=
git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6 h1:9Ci4a+yqtRdnj8JitXaRGntxeAkdFe+NltnR2ehl4vo=
git.mills.io/prologic/msgbus v0.1.13-0.20220329220338-7181b6df1bd6/go.mod h1:UyiNBmWbpsq7mtO+FHWoGwRiccPcT+EJGqT/idm/lfo=
git.mills.io/prologic/msgbus v0.1.13-0.20220402140515-7b71102aa813 h1:lMjKwVDctfeqnxMLHm/PQ3kngbedn6indnU6noInYWk=
git.mills.io/prologic/msgbus v0.1.13-0.20220402140515-7b71102aa813/go.mod h1:LitIrrXM81t/9+UNl0WN9B9lCeIfkOrf/Fee7CLf+7A=
git.mills.io/prologic/msgbus v0.1.13 h1:UkyexCHV2zL1mXZlOTVpQ8JE/aeBP4+7YesoHIgnrmc=
git.mills.io/prologic/msgbus v0.1.13/go.mod h1:LitIrrXM81t/9+UNl0WN9B9lCeIfkOrf/Fee7CLf+7A=
git.mills.io/prologic/msgbus v0.1.14 h1:2YAsjFwrUt+BHRit55LOQkXCC4E3v5Mk/c/rw/yThhk=
git.mills.io/prologic/msgbus v0.1.14/go.mod h1:LitIrrXM81t/9+UNl0WN9B9lCeIfkOrf/Fee7CLf+7A=
git.mills.io/prologic/msgbus v0.1.15-0.20220403155938-6bfb669347cc h1:lZt7rz3o/lIxk+LHzVQF6LMEaJhH+P8sMutlKrjDmg8=
git.mills.io/prologic/msgbus v0.1.15-0.20220403155938-6bfb669347cc/go.mod h1:9389uj/tLfr5qSbRFr9dDuUIKeuVv30tv0pdzLhgXnc=
git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1 h1:e6ZyAOFGLZJZYL2galNvfuNMqeQDdilmQ5WRBXCNL5s=
git.mills.io/prologic/observe v0.0.0-20210712230028-fc31c7aa2bd1/go.mod h1:/rNXqsTHGrilgNJYH/8wsIRDScyxXUhpbSdNbBatAKY=
git.mills.io/prologic/useragent v0.0.0-20210714100044-d249fe7921a0 h1:MojWEgZyiugUbgyjydrdSAkHlADnbt90dXyURRYFzQ4=
@ -406,8 +402,6 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
github.com/petermattis/goid v0.0.0-20220302125637-5f11c28912df h1:/B1Q9E4W1cmiwPQfC2vymWL7FXHCEsUzg8Rywl5avtQ=
github.com/petermattis/goid v0.0.0-20220302125637-5f11c28912df/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/petermattis/goid v0.0.0-20220331194723-8ee3e6ded87a h1:VXRRto5GMJPNfB7MNbUVoFhtxwoYjBEsIt/NpWg42U0=
github.com/petermattis/goid v0.0.0-20220331194723-8ee3e6ded87a/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@ -522,8 +516,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/redcon v1.4.1/go.mod h1:XwNPFbJ4ShWNNSA2Jazhbdje6jegTCcwFR6mfaADvHA=
github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I=
github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8=
github.com/tidwall/wal v1.1.7 h1:emc1TRjIVsdKKSnpwGBAcsAGg0767SvUk8+ygx7Bb+4=
github.com/tidwall/wal v1.1.7/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E=
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09 h1:QVxbx5l/0pzciWYOynixQMtUhPYC3YKD6EcUlOsgGqw=
github.com/timewasted/go-accept-headers v0.0.0-20130320203746-c78f304b1b09/go.mod h1:Uy/Rnv5WKuOO+PuDhuYLEpUiiKIZtss3z519uk67aF0=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -540,9 +546,13 @@ github.com/unrolled/render v1.4.1 h1:VdpMc2YkAOWzbmC/P2yoHhRDXgsaCQHcTJ1KK6SNCA4
github.com/unrolled/render v1.4.1/go.mod h1:cK4RSTTVdND5j9EYEc0LAMOvdG11JeiKjyjfyZRvV2w=
github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U=
github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
github.com/wblakecaldwell/profiler v0.0.0-20150908040756-6111ef1313a1 h1:Dz/PRieZRmOhDfOlkVpY1LYYIfNoTJjlDirAlagOr0s=
github.com/wblakecaldwell/profiler v0.0.0-20150908040756-6111ef1313a1/go.mod h1:3+0F8oLB1rQlbIcRAuqDgGdzNi9X69un/aPz4cUAFV4=
github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28=
@ -774,6 +784,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f h1:rlezHXNlxYWvBCzNses9Dlc7nGFaNMJeqLolcmQSSZY=
golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0 h1:PgUUmg0gNMIPY2WafhL/oLyQGw+kdTNPlVWOjltpp3w=
golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

@ -46,14 +46,6 @@ func (s *Server) IndexHandler() httprouter.Handle {
}
}
// InboxHandler ...
func (s *Server) InboxHandler() httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
// TODO: Implement filtering / blocking at the broker level
s.bus.ServeHTTP(w, r)
}
}
// AvatarHandler ...
func (s *Server) AvatarHandler() httprouter.Handle {
avatarsDir := filepath.Join(s.config.Data, avatarsPath)

@ -30,6 +30,7 @@ import (
const (
acmeDir = "acme"
logsDir = "logs"
servicesIdentity = "svc.key"
)
@ -342,9 +343,9 @@ func (s *Server) initRoutes() {
// Discovery
s.router.GET("/.well-known/salty/:config", s.ConfigHandler())
// Inbox
s.router.GET("/inbox/:inbox", s.InboxHandler())
s.router.POST("/inbox/:inbox", s.InboxHandler())
// Inbox Endpoint(s)
s.router.Handler(http.MethodGet, "/inbox/:inbox", http.StripPrefix("/inbox", s.bus))
s.router.Handler(http.MethodPost, "/inbox/:inbox", http.StripPrefix("/inbox", s.bus))
// Avatar Service
s.router.GET("/avatar/:hash", s.AvatarHandler())
@ -365,7 +366,13 @@ func NewServer(bind string, options ...Option) (*Server, error) {
return nil, fmt.Errorf("error validating config: %w", err)
}
bus := msgbus.New(nil)
bus, err := msgbus.NewMessageBus(
msgbus.WithLogPath(filepath.Join(config.Data, logsDir)),
)
if err != nil {
log.WithError(err).Error("error creating message bus")
return nil, err
}
db, err := NewStore(config.Store)
if err != nil {

BIN
internal/web/app.wasm (Stored with Git LFS)

Binary file not shown.

@ -17,27 +17,23 @@ func DefaultState() string {
type State struct {
sync.RWMutex
Indicies map[string]int
Indicies map[string]int64
}
func NewState() *State {
return &State{
Indicies: make(map[string]int),
Indicies: make(map[string]int64),
}
}
func (s *State) GetIndex(name string) int {
func (s *State) GetIndex(name string) int64 {
s.RLock()
defer s.RUnlock()
index, ok := s.Indicies[name]
if !ok {
return -1 // -1 is a special value meaning start from the beginning
}
return index
return s.Indicies[name]
}
func (s *State) SetIndex(name string, index int) {
func (s *State) SetIndex(name string, index int64) {
s.Lock()
defer s.Unlock()

Loading…
Cancel
Save