Add support for discovering peering pods #582

Merged
prologic merged 41 commits from pod_peers into master 6 months ago
  1. 1
      .gitignore
  2. 5
      Makefile
  3. 167
      internal/cache.go
  4. 364
      internal/cache_test.go
  5. 13
      internal/config.go
  6. 3
      internal/context.go
  7. 15
      internal/dispatcher.go
  8. 14
      internal/handlers.go
  9. 1
      internal/langs/active.en.toml
  10. 21
      internal/manage_handlers.go
  11. 3
      internal/server.go
  12. 4
      internal/theme/static/css/02-tabler-icons.css
  13. 2
      internal/theme/templates/base.html
  14. 3
      internal/theme/templates/info.html
  15. 25
      internal/theme/templates/managePeers.html
  16. 5
      internal/theme/templates/managePod.html
  17. 10
      internal/twtxt_handlers.go
  18. 133
      internal/utils.go
  19. 138
      internal/utils_test.go
  20. 15
      tools/check-pod-versions.sh

1
.gitignore vendored

@ -42,6 +42,7 @@
/internal/theme/static/js/yarn.min.js
bench-yarn.txt
coverage.out
.task/
/doc/

5
Makefile

@ -15,6 +15,7 @@ preflight:
deps:
@$(GOCMD) install github.com/tdewolff/minify/v2/cmd/minify@latest
@$(GOCMD) install github.com/nicksnyder/go-i18n/v2/goi18n@latest
@$(GOCMD) install github.com/astaxie/bat@latest
dev : DEBUG=1
dev : build
@ -64,6 +65,10 @@ release:
test:
@$(GOCMD) test -v -cover -race ./...
coverage:
@$(GOCMD) test -v -cover -race -cover -coverprofile=coverage.out ./...
@$(GOCMD) tool cover -html=coverage.out
bench: bench-yarn.txt
go test -race -benchtime=1x -cpu 16 -benchmem -bench "^(Benchmark)" git.mills.io/yarnsocial/yarn/types

167
internal/cache.go

@ -3,8 +3,10 @@ package internal
import (
"bytes"
"encoding/gob"
"encoding/json"
"fmt"
"io"
"mime"
"net/http"
"os"
"path/filepath"
@ -21,10 +23,12 @@ import (
const (
feedCacheFile = "cache"
feedCacheVersion = 17 // increase this if breaking changes occur to cache file.
feedCacheVersion = 18 // increase this if breaking changes occur to cache file.
localViewKey = "local"
discoverViewKey = "discover"
podInfoUpdateTTL = time.Hour * 24
)
// FilterFunc ...
@ -153,6 +157,31 @@ func (cached *Cached) Update(url, lastmodiied string, twts types.Twts) {
cached.LastModified = lastmodiied
}
type PodInfo struct {
Name string `json:"name"`
Description string `json:"description"`
SoftwareVersion string `json:"software_version"`
prologic marked this conversation as resolved
Review

Typo in "desc_ri_ption".

Typo in "desc_ri_ption".
// Maybe we store future data about other peer pods in the future?
// Right now the above is basically what is exposed now as the pod's name, description and what version of yarnd is running.
// This information will likely be used for Pod Owner/Operators to manage Image Domain Whitelisting between pods and internal
// automated operations like Pod Gossiping of Twts for things like Missing Root Twts for conversation views, etc.
// lastSeen records the timestamp of when we last saw this pod.
LastSeen time.Time `json:"-"`
// lastUpdated is used to periodically re-check the peering pod's /info endpoint in case of changes.
LastUpdated time.Time `json:"-"`
}
func (p *PodInfo) IsZero() bool {
return (p == nil) || (p.Name == "" && p.SoftwareVersion == "")
}
func (p *PodInfo) ShouldRefresh() bool {
return time.Since(p.LastUpdated) > podInfoUpdateTTL
}
// Cache ...
type Cache struct {
mu sync.RWMutex
@ -163,6 +192,7 @@ type Cache struct {
List *Cached
Map map[string]types.Twt
Peers map[string]*PodInfo
Feeds map[string]*Cached
Views map[string]*Cached
}
@ -171,6 +201,7 @@ func NewCache(conf *Config) *Cache {
return &Cache{
conf: conf,
Map: make(map[string]types.Twt),
Peers: make(map[string]*PodInfo),
Feeds: make(map[string]*Cached),
Views: make(map[string]*Cached),
}
@ -247,6 +278,122 @@ func LoadCache(conf *Config) (*Cache, error) {
return cache, nil
}
// DetectPodFromRequest ...
func (cache *Cache) DetectPodFromRequest(req *http.Request) error {
twtxtUA, err := ParseUserAgent(req.UserAgent())
if err != nil {
log.WithError(err).Warnf("error parsing Twtxt User-Agent '%s'", req.UserAgent())
return nil
}
if !twtxtUA.IsPod() {
log.Debugf("%s is not a pod!", req.UserAgent())
return nil
}
podBaseURL := twtxtUA.PodBaseURL()
log.Debugf("podBaseURL: %#v", podBaseURL)
if podBaseURL == "" {
log.Debugf("cannot find a valid pod base URL from %s", req.UserAgent())
return nil
}
cache.mu.RLock()
oldPodInfo, hasSeen := cache.Peers[podBaseURL]
cache.mu.RUnlock()
if hasSeen && !oldPodInfo.ShouldRefresh() {
log.Debugf("already seen pod %s", podBaseURL)
// This might in fact race if another goroutine would have fetched the
// pod info and updated the cache between our check above and the
// update here. However, since we're only setting a timestamp when
// we've last seen the peering pod, this should not be a problem at
// all. We just override it a fraction of a second later. Doesn't harm
// anything.
cache.mu.Lock()
oldPodInfo.LastSeen = time.Now()
cache.mu.Unlock()
return nil
}
// Set an empty &PodInfo{} to avoid multiple concurrent calls from making
// multiple callbacks to peering pods unncessarily for Multi-User pods and
// guard against race from other goroutine doing the same thing.
cache.mu.Lock()
oldPodInfo, hasSeen = cache.Peers[podBaseURL]
if hasSeen && !oldPodInfo.ShouldRefresh() {
cache.mu.Unlock()
log.Debugf("already seen pod %s", podBaseURL)
return nil
}
cache.Peers[podBaseURL] = &PodInfo{}
cache.mu.Unlock()
resetDummyPodInfo := func() {
cache.mu.Lock()
if oldPodInfo.IsZero() {
delete(cache.Peers, podBaseURL)
} else {
cache.Peers[podBaseURL] = oldPodInfo
}
cache.mu.Unlock()
}
headers := make(http.Header)
headers.Set("Accept", "application/json")
res, err := Request(cache.conf, http.MethodGet, podBaseURL+"/info", headers)
if err != nil {
resetDummyPodInfo()
log.WithError(err).Errorf("error making /info request to pod running at %s", podBaseURL)
return err
}
defer res.Body.Close()
if res.StatusCode / 100 != 2 {
resetDummyPodInfo()
log.Errorf("HTTP %s response for /info of pod running at %s", res.Status, podBaseURL)
return fmt.Errorf("non-success HTTP %s response for %s/info", res.Status, podBaseURL)
}
if ctype := res.Header.Get("Content-Type"); ctype != "" {
mediaType, _, err := mime.ParseMediaType(ctype)
if err != nil {
resetDummyPodInfo()
log.WithError(err).Errorf("error parsing content type header '%s' for /info of pod running at %s", ctype, podBaseURL)
return err
}
if mediaType != "application/json" {
resetDummyPodInfo()
log.Errorf("non-JSON response '%s' for /info of pod running at %s", ctype, podBaseURL)
return fmt.Errorf("non-JSON response content type '%s' for %s/info", ctype, podBaseURL)
}
}
data, err := io.ReadAll(res.Body)
if err != nil {
resetDummyPodInfo()
log.WithError(err).Errorf("error reading response body for /info of pod running at %s", podBaseURL)
return err
}
var podInfo PodInfo
if err := json.Unmarshal(data, &podInfo); err != nil {
resetDummyPodInfo()
log.WithError(err).Errorf("error decoding response body for /info of pod running at %s", podBaseURL)
return err
}
podInfo.LastSeen = time.Now()
podInfo.LastUpdated = time.Now()
cache.mu.Lock()
cache.Peers[podBaseURL] = &podInfo
cache.mu.Unlock()
return nil
}
// FetchTwts ...
func (cache *Cache) FetchTwts(conf *Config, archive Archiver, feeds types.Feeds, publicFollowers map[types.Feed][]string) {
stime := time.Now()
@ -636,6 +783,24 @@ func (cache *Cache) UpdateFeed(url, lastmodified string, twts types.Twts) {
}
}
// GetPeers ...
func (cache *Cache) GetPeers() map[string]*PodInfo {
cache.mu.RLock()
cachedPeers := cache.Peers
cache.mu.RUnlock()
peers := make(map[string]*PodInfo)
for k, v := range cachedPeers {
if k == "" || v.IsZero() {
continue
}
peers[k] = v
}
return peers
}
// GetAll ...
func (cache *Cache) GetAll(refresh bool) types.Twts {
cache.mu.RLock()

364
internal/cache_test.go

@ -0,0 +1,364 @@
package internal
import (
"encoding/json"
"fmt"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var peerCfg = &Config{requestTimeout: 100 * time.Millisecond}
func newNoCallbackExpectedServer(t *testing.T) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
t.Fatal("expected callback URL not being called")
}))
}
func newCallbackExpectedServerWithResponse(t *testing.T, reply func(w http.ResponseWriter)) (*httptest.Server, func()) {
called := make(chan bool, 1)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
assert.Equal(t, "/info", req.RequestURI, "callback URI mismatch")
reply(w)
called <- true
}))
cleanup := func() {
server.Close()
select {
case <-called:
return
default:
t.Fatal("expected callback URL being called, but was not")
}
}
return server, cleanup
}
func newCallbackExpectedServerWithNewPodInfo(t *testing.T) (*httptest.Server, func()) {
response, err := json.Marshal(PodInfo{
Name: "new name",
Description: "new description",
SoftwareVersion: "0.9001.23@7654321",
})
require.NoError(t, err, "marshalling pod info for callback failed")
return newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
w.Write(response)
})
}
func randomPort(t *testing.T) int {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
if listener, err = net.Listen("tcp6", "[::1]:0"); err != nil {
t.Fatalf("failed to listen on a port: %v", err)
return 0
}
}
defer listener.Close()
return listener.Addr().(*net.TCPAddr).Port
}
func newRequestWithUA(ua string) *http.Request {
req, err := http.NewRequest("GET", "http://localhost/user/foo/twtxt.txt", nil)
if err != nil {
panic("creating test HTTP request failed")
}
req.Header.Set("User-Agent", ua)
return req
}
func newCacheWithPodInfo(podBaseURL string, lastSeenAndUpdated time.Time) *Cache {
cache := NewCache(peerCfg)
cache.Peers[podBaseURL] = &PodInfo{
Name: "old name",
Description: "old description",
SoftwareVersion: "0.42.0@1234567",
LastSeen: lastSeenAndUpdated,
LastUpdated: lastSeenAndUpdated,
}
return cache
}
func assertAfterOrAt(t *testing.T, expected, actual time.Time, msg string) {
if actual.Before(actual) {
t.Fatalf("%s is not after or at %s: %s", actual.Format(time.RFC3339), expected.Format(time.RFC3339), msg)
}
}
func assertBefore(t *testing.T, expected, actual time.Time, msg string) {
if !actual.Before(expected) {
t.Fatalf("%s is not before %s: %s", actual.Format(time.RFC3339), expected.Format(time.RFC3339), msg)
}
}
func assertPodInfoNotInserted(t *testing.T, cache *Cache) {
assert.Empty(t, cache.Peers, "cached peers should not have been updated")
}
func assertPodInfoNotUpdatedExceptLastSeen(t *testing.T, cache *Cache, podBaseURL string,
expectedNowReferenceBeforeCallForLastSeen, expectedLastUpdated time.Time) {
podInfo, ok := cache.Peers[podBaseURL]
require.True(t, ok, "cached pod info should not have been removed from the cache")
require.NotNil(t, podInfo, "cached pod info should not have been removed from the cache")
assert.Equal(t, "old name", podInfo.Name, "cached pod name should not have been updated")
assert.Equal(t, "old description", podInfo.Description, "cached pod description shold not have been updated")
assert.Equal(t, "0.42.0@1234567", podInfo.SoftwareVersion, "cached pod software version should not have been updated")
assertAfterOrAt(t, expectedNowReferenceBeforeCallForLastSeen, podInfo.LastSeen,
"cached last seen should have been updated to current point in time")
assertBefore(t, expectedNowReferenceBeforeCallForLastSeen.Add(10 * time.Second) /* allow for a little clock skew */,
podInfo.LastSeen, "cached last seen should not have been updated to be in the future")
assert.Equal(t, expectedLastUpdated, podInfo.LastUpdated, "cached pod last updated should not have been updated")
}
func assertPodInfoUpdated(t *testing.T, cache *Cache, podBaseURL string,
expectedNowReferenceBeforeCallForLastSeenAndUpdated time.Time) {
podInfo, ok := cache.Peers[podBaseURL]
require.True(t, ok, "cached pod info should have been inserted into/not removed from the cache")
require.NotNil(t, podInfo, "cached pod info should have been inserted into/not removed from the cache")
assert.Equal(t, "new name", podInfo.Name, "cached pod name should have been updated")
assert.Equal(t, "new description", podInfo.Description, "cached pod description shold have been updated")
assert.Equal(t, "0.9001.23@7654321", podInfo.SoftwareVersion, "cached pod software version should have been updated")
assertAfterOrAt(t, expectedNowReferenceBeforeCallForLastSeenAndUpdated, podInfo.LastSeen,
"cached last seen should have been updated to current point in time")
assertBefore(t, expectedNowReferenceBeforeCallForLastSeenAndUpdated.Add(10 * time.Second) /* allow for a little clock skew */,
podInfo.LastSeen, "cached last seen should not have been updated to be in the future")
assertAfterOrAt(t, expectedNowReferenceBeforeCallForLastSeenAndUpdated, podInfo.LastUpdated,
"cached last updated should have been updated to current point in time")
assertBefore(t, expectedNowReferenceBeforeCallForLastSeenAndUpdated.Add(10 * time.Second) /* allow for a little clock skew */,
podInfo.LastUpdated, "cached last updated should not have been updated to be in the future")
}
func TestCache_DetectPodFromRequest_whenNonTwtxtUserAgent_thenDoNothing(t *testing.T) {
server := newNoCallbackExpectedServer(t)
defer server.Close()
cache := NewCache(peerCfg)
req := newRequestWithUA("Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)")
assert.NoError(t, cache.DetectPodFromRequest(req), "detecting pod failed")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenNonYarnTwtxtUserAgent_thenDoNothing(t *testing.T) {
server := newNoCallbackExpectedServer(t)
defer server.Close()
cache := NewCache(peerCfg)
req := newRequestWithUA("twtxt/1.2.3 (+https://example.com/twtxt.txt; @foo)")
assert.NoError(t, cache.DetectPodFromRequest(req), "detecting pod failed")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenWithinConfiguredTTL_thenDoNotCallbackButUpdateLastSeen(t *testing.T) {
server := newNoCallbackExpectedServer(t)
defer server.Close()
lastSeenAndUpdated := time.Now().Add(-3 * time.Minute)
cache := newCacheWithPodInfo(server.URL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.42.0@1234567 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
assert.NoError(t, cache.DetectPodFromRequest(req), "detecting pod failed")
assertPodInfoNotUpdatedExceptLastSeen(t, cache, server.URL, now, lastSeenAndUpdated)
}
func TestCache_DetectPodFromRequest_whenPodNeverSeen_thenCallbackAndPopulateCache(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithNewPodInfo(t)
defer cleanup()
cache := NewCache(peerCfg)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
assert.NoError(t, cache.DetectPodFromRequest(req), "detecting pod failed")
assertPodInfoUpdated(t, cache, server.URL, now)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenOutsideConfiguredTTL_thenCallbackAndUpdateCache(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithNewPodInfo(t)
defer cleanup()
lastSeenAndUpdated := time.Now().Add(-25 * time.Hour)
cache := newCacheWithPodInfo(server.URL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
assert.NoError(t, cache.DetectPodFromRequest(req), "detecting pod failed")
assertPodInfoUpdated(t, cache, server.URL, now)
}
func TestCache_DetectPodFromRequest_whenPodNeverSeenAndCallbackNotReplying_thenReturnErrorAndDoNothing(t *testing.T) {
cache := NewCache(peerCfg)
serverURL := fmt.Sprintf("http://localhost:%d", randomPort(t))
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", serverURL))
err := cache.DetectPodFromRequest(req)
assert.Error(t, err, "detecting pod should have failed")
assert.Contains(t, err.Error(), serverURL + "/info", "error message should contain callback URL")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenAndCallbackNotReplying_thenReturnErrorAndUpdateOnlyLastSeen(t *testing.T) {
serverURL := fmt.Sprintf("http://localhost:%d", randomPort(t))
lastSeenAndUpdated := time.Now().Add(-25 * time.Hour)
cache := newCacheWithPodInfo(serverURL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", serverURL))
now := time.Now()
err := cache.DetectPodFromRequest(req)
assert.Error(t, err, "detecting pod should have failed")
assert.Contains(t, err.Error(), serverURL + "/info", "error message should contain callback URL")
assertPodInfoNotUpdatedExceptLastSeen(t, cache, serverURL, now, lastSeenAndUpdated)
}
func TestCache_DetectPodFromRequest_whenPodNeverSeenAndCallbackReplyingWithHTTPNon200_thenReturnErrorAndDoNothing(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.WriteHeader(404)
w.Write([]byte("I'm a too old yarnd version which does not support the new /info endpoint"))
})
defer cleanup()
cache := NewCache(peerCfg)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
err := cache.DetectPodFromRequest(req)
assert.EqualError(t, err, fmt.Sprintf("non-success HTTP 404 Not Found response for %s/info", server.URL),
"detecting pod should have failed")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenAndCallbackReplyingWithHTTPNon200_thenReturnErrorAndUpdateOnlyLastSeen(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.WriteHeader(404)
w.Write([]byte("I'm a too old yarnd version which does not support the new /info endpoint"))
})
defer cleanup()
lastSeenAndUpdated := time.Now().Add(-25 * time.Hour)
cache := newCacheWithPodInfo(server.URL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
err := cache.DetectPodFromRequest(req)
assert.EqualError(t, err, fmt.Sprintf("non-success HTTP 404 Not Found response for %s/info", server.URL),
"detecting pod should have failed")
assertPodInfoNotUpdatedExceptLastSeen(t, cache, server.URL, now, lastSeenAndUpdated)
}
func TestCache_DetectPodFromRequest_whenPodNeverSeenAndCallbackReplyingWithInvalidContentType_thenReturnErrorAndDoNothing(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "improper content type")
w.WriteHeader(200)
w.Write([]byte("whoops"))
})
defer cleanup()
cache := NewCache(peerCfg)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
err := cache.DetectPodFromRequest(req)
assert.EqualError(t, err, "mime: expected slash after first token", "detecting pod should have failed")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenAndCallbackReplyingWithInvalidContentType_thenReturnErrorAndUpdateOnlyLastSeen(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "improper content type")
w.WriteHeader(200)
w.Write([]byte("whoops"))
})
defer cleanup()
lastSeenAndUpdated := time.Now().Add(-25 * time.Hour)
cache := newCacheWithPodInfo(server.URL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
err := cache.DetectPodFromRequest(req)
assert.EqualError(t, err, "mime: expected slash after first token", "detecting pod should have failed")
assertPodInfoNotUpdatedExceptLastSeen(t, cache, server.URL, now, lastSeenAndUpdated)
}
func TestCache_DetectPodFromRequest_whenPodNeverSeenAndCallbackReplyingWithNonJSONContentType_thenReturnErrorAndDoNothing(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
w.WriteHeader(200)
w.Write([]byte("that's no JSON as indicated in the Content-Type header"))
})
defer cleanup()
cache := NewCache(peerCfg)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
err := cache.DetectPodFromRequest(req)
assert.EqualError(t, err, fmt.Sprintf("non-JSON response content type 'text/plain; charset=UTF-8' for %s/info", server.URL),
"detecting pod should have failed")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenAndCallbackReplyingWithNonJSONContentType_thenReturnErrorAndUpdateOnlyLastSeen(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
w.WriteHeader(200)
w.Write([]byte("that's no JSON as indicated in the Content-Type header"))
})
defer cleanup()
lastSeenAndUpdated := time.Now().Add(-25 * time.Hour)
cache := newCacheWithPodInfo(server.URL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
err := cache.DetectPodFromRequest(req)
assert.EqualError(t, err, fmt.Sprintf("non-JSON response content type 'text/plain; charset=UTF-8' for %s/info", server.URL),
"detecting pod should have failed")
assertPodInfoNotUpdatedExceptLastSeen(t, cache, server.URL, now, lastSeenAndUpdated)
}
func TestCache_DetectPodFromRequest_whenPodNeverSeenAndCallbackReplyingWithJSONGarbage_thenReturnErrorAndDoNothing(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
w.Write([]byte("this is no JSON"))
})
defer cleanup()
cache := NewCache(peerCfg)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
err := cache.DetectPodFromRequest(req)
assert.Error(t, err, "detecting pod should have failed")
assert.Contains(t, err.Error(), "invalid character", "error message should say something about decoding error")
assertPodInfoNotInserted(t, cache)
}
func TestCache_DetectPodFromRequest_whenPodAlreadySeenAndCallbackReplyingWithJSONGarbage_thenReturnErrorAndUpdateOnlyLastSeen(t *testing.T) {
server, cleanup := newCallbackExpectedServerWithResponse(t, func(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
w.Write([]byte("this is no JSON"))
})
defer cleanup()
lastSeenAndUpdated := time.Now().Add(-25 * time.Hour)
cache := newCacheWithPodInfo(server.URL, lastSeenAndUpdated)
req := newRequestWithUA(fmt.Sprintf("yarnd/0.9001.23@7654321 (+%s/user/bar/twtxt.txt; @bar)", server.URL))
now := time.Now()
err := cache.DetectPodFromRequest(req)
assert.Error(t, err, "detecting pod should have failed")
assert.Contains(t, err.Error(), "invalid character", "error message should say something about decoding error")
assertPodInfoNotUpdatedExceptLastSeen(t, cache, server.URL, now, lastSeenAndUpdated)
}

13
internal/config.go

@ -136,6 +136,9 @@ type Config struct {
DisplayDatesInTimezone string
DisplayTimePreference string
OpenLinksInPreference string
// requestTimeout defines the timeout for outgoing HTTP requests.
requestTimeout time.Duration
}
var _ types.FmtOpts = (*Config)(nil)
@ -272,6 +275,15 @@ func (c *Config) TemplatesFS() fs.FS {
return os.DirFS(filepath.Join(c.Theme, "templates"))
}
// RequestTimeout returns the configured timeout for outgoing HTTP requests. If
// not defined, it defaults to 30 seconds.
func (c *Config) RequestTimeout() time.Duration {
if c.requestTimeout == 0 {
return 30 * time.Second
}
return c.requestTimeout
}
// LoadSettings loads pod settings from the given path
func LoadSettings(path string) (*Settings, error) {
var settings Settings
@ -315,3 +327,4 @@ func (s *Settings) Save(path string) error {
return f.Close()
}

3
internal/context.go

@ -101,6 +101,9 @@ type Context struct {
DiscoverUpdatedAt time.Time
LastMentionedAt time.Time
// Discovered Pods peering with us
Peers map[string]*PodInfo
// Search
SearchQuery string

15
internal/dispatcher.go

@ -2,11 +2,14 @@ package internal
import (
"errors"
sync "github.com/sasha-s/go-deadlock"
)
// Dispatcher maintains a pool for available workers
// and a task queue that workers will process
type Dispatcher struct {
sync.RWMutex
maxWorkers int
maxQueue int
workers []*Worker
@ -31,6 +34,9 @@ func NewDispatcher(maxWorkers int, maxQueue int) *Dispatcher {
// Then, it starts a select loop to wait for tasks to be dispatched
// to available workers
func (d *Dispatcher) Start() {
d.Lock()
defer d.Unlock()
d.workers = []*Worker{}
d.workerPool = make(chan chan Task, d.maxWorkers)
d.taskQueue = make(chan Task, d.maxQueue)
@ -63,6 +69,9 @@ func (d *Dispatcher) Start() {
// Stop ends execution for all workers and closes all channels, then removes
// all workers
func (d *Dispatcher) Stop() {
d.Lock()
defer d.Unlock()
if !d.active {
return
}
@ -79,6 +88,9 @@ func (d *Dispatcher) Stop() {
// Lookup returns the matching `Task` given its id
func (d *Dispatcher) Lookup(id string) (Task, bool) {
d.RLock()
defer d.RUnlock()
task, ok := d.taskMap[id]
return task, ok
}
@ -86,6 +98,9 @@ func (d *Dispatcher) Lookup(id string) (Task, bool) {
// Dispatch pushes the given task into the task queue.
// The first available worker will perform the task
func (d *Dispatcher) Dispatch(task Task) (string, error) {
d.Lock()
defer d.Unlock()
if !d.active {
return "", errors.New("dispatcher is not active")
}

14
internal/handlers.go

@ -739,11 +739,15 @@ func (s *Server) SyndicationHandler() httprouter.Handle {
}
}
// PodVersionHandler ...
func (s *Server) PodVersionHandler() httprouter.Handle {
// PodInfoHandler ...
func (s *Server) PodInfoHandler() httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
if r.Header.Get("Accept") == "application/json" {
data, err := json.Marshal(s.config.Version)
data, err := json.Marshal(PodInfo{
Name: s.config.Name,
Description: s.config.Description,
SoftwareVersion: s.config.Version.FullVersion,
})
if err != nil {
log.WithError(err).Error("error serializing pod version response")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@ -754,7 +758,7 @@ func (s *Server) PodVersionHandler() httprouter.Handle {
_, _ = w.Write(data)
} else {
ctx := NewContext(s, r)
s.render("version", w, ctx)
s.render("info", w, ctx)
}
}
}
@ -762,7 +766,7 @@ func (s *Server) PodVersionHandler() httprouter.Handle {
// PodConfigHandler ...
func (s *Server) PodConfigHandler() httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
data, err := json.Marshal(s.config.Settings())
data, err := json.Marshal(s.config)
if err != nil {
log.WithError(err).Error("error serializing pod config response")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)

1
internal/langs/active.en.toml

@ -148,6 +148,7 @@ ManageFeedFormUpdate = "Update"
ManageFeedSummary = "Manage <b>{{ .Username}}</b> details"
ManageFeedTitle = "Manage feed"
ManagePodLinkTitle = "Manage Pod"
ManagePeersLinkTitle = "Manage Peers"
ManageUsersLinkTitle = "Manage Users"
ManageRefreshCacheTitle = "Refresh Cache"
MeLinkTitle = "me"

21
internal/manage_handlers.go

@ -158,7 +158,6 @@ func (s *Server) ManageUsersHandler() httprouter.Handle {
}
s.render("manageUsers", w, ctx)
}
}
@ -500,3 +499,23 @@ func (s *Server) RefreshCacheHandler() httprouter.Handle {
s.render("error", w, ctx)
}
}
// ManagePeersHandler ...
func (s *Server) ManagePeersHandler() httprouter.Handle {
isAdminUser := IsAdminUserFactory(s.config)
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
ctx := NewContext(s, r)
if !isAdminUser(ctx.User) {
ctx.Error = true
ctx.Message = "You are not a Pod Owner!"
s.render("403", w, ctx)
return
}
ctx.Peers = s.cache.GetPeers()
s.render("managePeers", w, ctx)
}
}

3
internal/server.go

@ -648,9 +648,10 @@ func (s *Server) initRoutes() {
s.router.GET("/settings", httproutermiddleware.Handler("settings", s.am.MustAuth(s.SettingsHandler()), mdlw))
s.router.POST("/settings", httproutermiddleware.Handler("settings", s.am.MustAuth(s.SettingsHandler()), mdlw))
s.router.GET("/version", httproutermiddleware.Handler("version", s.PodVersionHandler(), mdlw))
s.router.GET("/info", httproutermiddleware.Handler("info", s.PodInfoHandler(), mdlw))
s.router.GET("/config", httproutermiddleware.Handler("config", s.am.MustAuth(s.PodConfigHandler()), mdlw))
s.router.GET("/manage/pod", httproutermiddleware.Handler("manage_pod", s.ManagePodHandler(), mdlw))
s.router.GET("/manage/peers", httproutermiddleware.Handler("manage_peers", s.ManagePeersHandler(), mdlw))
s.router.POST("/manage/pod", httproutermiddleware.Handler("manage_pod", s.ManagePodHandler(), mdlw))
s.router.GET("/manage/refreshcache", httproutermiddleware.Handler("manage_refreshcache", s.RefreshCacheHandler(), mdlw))

4
internal/theme/static/css/02-tabler-icons.css

@ -208,3 +208,7 @@
.ti-volume-3:before {
content: "\eb50";
}
.ti-affiliate:before {
content: '\edff';
}

2
internal/theme/templates/base.html

@ -101,7 +101,7 @@
<footer class="container">
<div class="footer-copyright">
Running <a href="https://git.mills.io/yarnsocial/yarn" target="_blank">yarnd</a>
<a href="/version">{{ .SoftwareVersion.FullVersion }}</a> &mdash;
<a href="/info">{{ .SoftwareVersion.FullVersion }}</a> &mdash;
a <a href="https://yarn.social" target="_blank">Yarn.social</a> pod.
</div>
<div class="footer-menu">

3
internal/theme/templates/version.html → internal/theme/templates/info.html

@ -1,9 +1,10 @@
{{ define "content" }}
<article class="container-fluid">
<hgroup>
<h2>{{ .InstanceName }} Version Information</h2>
<h2>{{ .InstanceName }} Information</h2>
</hgroup>
<table>
<tr><td>Name</td><td>{{ .InstanceName }}</td></tr>
<tr><td>Software</td><td>{{ .SoftwareVersion.Software }}</td></tr>
<tr><td>FullVersion</td><td>{{ .SoftwareVersion.FullVersion }}</td></tr>
<tr><td>Version</td><td>{{ .SoftwareVersion.Version }}</td></tr>

25
internal/theme/templates/managePeers.html

@ -0,0 +1,25 @@
{{ define "content" }}
<article class="container-fluid">
<hgroup>
<h2>Discovered Peering Pods</h2>
</hgroup>
<table>
<tr>
<th>Name</th>
<th>Description</th>
<th>Pod Version</th>
<th><abbr title="When this pod received the last feed fetch request from the peering pod">Last Seen<abbr></th>
<th><abbr title="When this pod fetched the peering pod's information the last time (once a day)">Last Updated</abbr></th>
</tr>
{{ range $k, $v := $.Peers }}
<tr>
<td><a href="{{ $k }}">{{ $v.Name }}</a></td>
<td><small>{{ $v.Description | abbrev 60 }}</small></td>
<td><small>{{ $v.SoftwareVersion }}</small></td>
<td><small>{{ $v.LastSeen | time }}</small></td>
<td><small>{{ $v.LastUpdated | time }}</small></td>
</tr>
{{ end }}
</table>
</article>
{{ end }}

5
internal/theme/templates/managePod.html

@ -6,8 +6,9 @@
<h3>Administer your Pod and update settings here</h3>
</hgroup>
<div class="manage-users">
<a href="/manage/users"><i class="ti ti-users"></i> Manage Pod Users</a><br /><br />
<a href="/manage/refreshcache"><i class="ti ti-rotate-clockwise-2"></i> Refresh Pod Cache</a>
<a href="/manage/peers"><i class="ti ti-affiliate"></i> Manage Peers</a><br /><br />
<a href="/manage/users"><i class="ti ti-users"></i> Manage Users</a><br /><br />
<a href="/manage/refreshcache"><i class="ti ti-rotate-clockwise-2"></i> Refresh Cache</a>
</div>
<form action="/manage/pod" enctype="multipart/form-data" method="POST">
<input type="hidden" name="csrf_token" value="{{ $.CSRFToken }}">

10
internal/twtxt_handlers.go

@ -21,7 +21,7 @@ const defaultPreambleTemplate = `# Twtxt is an open, distributed microblogging p
#
# Learn more about twtxt at https://github.com/buckket/twtxt
#
# This is hosted by a Yarn.social pod {{ .InstanceName }} running yarnd v{{ .SoftwareVersion.FullVersion }}
# This is hosted by a Yarn.social pod {{ .InstanceName }} running yarnd {{ .SoftwareVersion.FullVersion }}
# Learn more about Yarn.social at https://yarn.social
#
# nick = {{ .Profile.Username }}
@ -43,6 +43,10 @@ const defaultPreambleTemplate = `# Twtxt is an open, distributed microblogging p
// TwtxtHandler ...
func (s *Server) TwtxtHandler() httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
s.tasks.DispatchFunc(func() error {
return s.cache.DetectPodFromRequest(r)
})
ctx := NewContext(s, r)
nick := NormalizeUsername(p.ByName("nick"))
@ -68,8 +72,8 @@ func (s *Server) TwtxtHandler() httprouter.Handle {
return
}
ua, _ := ParseTwtxtUserAgent(r.UserAgent())
if ua != nil {
twtxtUA, _ := ParseUserAgent(r.UserAgent())
if ua, ok := twtxtUA.(*SingleUserAgent); ok {
var (
user *User
feed *Feed

133
internal/utils.go

@ -71,8 +71,6 @@ const (
maxFeedNameLength = 25 // avg 4.7 chars per word in English so ~5 words
maxTwtContextLength = 140
requestTimeout = time.Second * 30
DayAgo = time.Hour * 24
WeekAgo = DayAgo * 7
MonthAgo = DayAgo * 30
@ -110,9 +108,10 @@ var (
append([]string{}, specialUsernames...),
automatedFeeds...)
validFeedName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*$`)
validUsername = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]+$`)
userAgentRegex = regexp.MustCompile(`(.+) \(\+(https?://\S+/\S+); @(\S+)\)`)
validFeedName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*$`)
validUsername = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]+$`)
singleUserUARegex = regexp.MustCompile(`(.+) \(\+(https?://\S+/\S+); @(\S+)\)`)
multiUserUARegex = regexp.MustCompile(`(.+) \(~(https?://\S+\/\S+); contact=(https?://\S+)\)`)
ErrInvalidFeedName = errors.New("error: invalid feed name")
ErrBadRequest = errors.New("error: request failed with non-200 response")
@ -327,7 +326,7 @@ func Request(conf *Config, method, url string, headers http.Header) (*http.Respo
req.Header = headers
client := http.Client{
Timeout: requestTimeout,
Timeout: conf.RequestTimeout(),
}
res, err := client.Do(req)
@ -1033,34 +1032,68 @@ func (u URI) String() string {
return fmt.Sprintf("%s://%s", u.Type, u.Path)
}
type TwtxtUserAgent struct {
// TwtxtUserAgent ...
type TwtxtUserAgent interface {
fmt.Stringer
// IsPod returns true if the Twtxt client's User-Agent appears to be a Yarn.social pod (single or multi-user).
IsPod() bool
// PodBaseURL returns the base URL of the client's User-Agent if it appears to be a Yarn.social pod (single or multi-user).
PodBaseURL() string
// IsPublicURL returns true if the Twtxt client's User-Agent is from what appears to be the public internet.
IsPublicURL() bool
}
// TwtxtUserAgent interface guards
var (
_ TwtxtUserAgent = (*SingleUserAgent)(nil)
_ TwtxtUserAgent = (*MultiUserAgent)(nil)
)
// twtxtUserAgent is a base class for both single and multi-user Twtxt User Agents.
type twtxtUserAgent struct {
Client string
Nick string
URL string
}
func (ua TwtxtUserAgent) String() string {
// twtxt/<version> (+<source.url>; @<source.nick>)
return fmt.Sprintf("%s (+%s; @%s)", ua.Client, ua.URL, ua.Nick)
func (ua *twtxtUserAgent) IsPod() bool {
return strings.HasPrefix(ua.Client, "yarnd/")
}
func (ua TwtxtUserAgent) IsPublicURL() bool {
u, err := url.Parse(ua.URL)
func (ua *twtxtUserAgent) podBaseURL(uri, relativeURLToTrim string) string {
if !ua.IsPod() {
return ""
}
u, err := url.Parse(uri)
if err != nil {
log.WithError(err).Warnf("error parsing User-Agent URL: %s", uri)
return ""
}
// Throw away the trailing part of the URL to get the base URL for this
// yarnd instance. It might serve from a subdirectory, so we cannot simply
// cut off the complete path.
rel, _ := url.Parse(relativeURLToTrim)
return NormalizeURL(u.ResolveReference(rel).String())
}
func (ua *twtxtUserAgent) isPublicURL(uri, userAgent string) bool {
u, err := url.Parse(uri)
if err != nil {
log.WithError(err).Warn("error parsing User-Agent URL")
return false
}
hostname := u.Hostname()
ips, err := net.LookupIP(hostname)
ips, err := net.LookupIP(u.Hostname())
if err != nil {
log.WithError(err).Warn("error looking up User-Agent IP")
return false
}
if len(ips) == 0 {
log.Warnf("error User-Agent lookup failed for %s or has no resolvable IP", ua.String())
log.Warnf("User-Agent lookup failed for %s or has no resolvable IP", userAgent)
return false
}
@ -1079,18 +1112,66 @@ func (ua TwtxtUserAgent) IsPublicURL() bool {
return !ipip.IsPrivate(ip)
}
func ParseTwtxtUserAgent(ua string) (*TwtxtUserAgent, error) {
match := userAgentRegex.FindStringSubmatch(ua)
// SingleUserAgent is a single Twtxt User Agent whether it be `tt`, `jenny` or a single-user `yarnd` client.
type SingleUserAgent struct {
twtxtUserAgent
Nick string
URL string
}
func (ua *SingleUserAgent) String() string {
// <client>/<version> (+<source.url>; @<source.nick>)
return fmt.Sprintf("%s (+%s; @%s)", ua.Client, ua.URL, ua.Nick)
}
func (ua *SingleUserAgent) PodBaseURL() string {
// get rid of the trailing '/user/foo/twtxt.txt'
return ua.podBaseURL(ua.URL, "../..")
}
func (ua *SingleUserAgent) IsPublicURL() bool {
return ua.isPublicURL(ua.URL, ua.String())
}
// MultiUserAgent is a multi-user Twtxt client, currently only `yarnd` is such a client.
type MultiUserAgent struct {
twtxtUserAgent
WhoFollowsURL string
SupportURL string
}
func (ua *MultiUserAgent) String() string {
// <client>/<version> (~<whoFollowsURL>; contact=<supportURL>)
return fmt.Sprintf("%s (~%s; contact=%s)", ua.Client, ua.WhoFollowsURL, ua.SupportURL)
}
func (ua *MultiUserAgent) PodBaseURL() string {
// get rid of the trailing '/whoFollows?followers=42&token=abc'
return ua.podBaseURL(ua.WhoFollowsURL, "./")
}
func (ua *MultiUserAgent) IsPublicURL() bool {
return ua.isPublicURL(ua.WhoFollowsURL, ua.String())
}
func ParseUserAgent(ua string) (TwtxtUserAgent, error) {
if match := singleUserUARegex.FindStringSubmatch(ua); match != nil {
return &SingleUserAgent{
twtxtUserAgent: twtxtUserAgent{Client: match[1]},
URL: match[2],
Nick: match[3],
}, nil
}
match := multiUserUARegex.FindStringSubmatch(ua)
if match == nil {
return nil, ErrInvalidUserAgent
}
// TODO: Add support for Multi-user UserAgent(s)
return &TwtxtUserAgent{
Client: match[1],
URL: match[2],
Nick: match[3],
return &MultiUserAgent{
twtxtUserAgent: twtxtUserAgent{Client: match[1]},
WhoFollowsURL: match[2],
SupportURL: match[3],
}, nil
}

138
internal/utils_test.go

@ -13,38 +13,142 @@ import (
"github.com/stretchr/testify/assert"
)
func TestParseTwtxtUserAgent(t *testing.T) {
func TestParseUserAgent(t *testing.T) {
testCases := []struct {
name string
ua string
err error
expected *TwtxtUserAgent
expected TwtxtUserAgent
}{
{
ua: `Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)`,
err: ErrInvalidUserAgent,
expected: nil,
name: "non-Twtxt User Agent",
ua: `Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)`,
err: ErrInvalidUserAgent,
},
{
ua: `twtxt/1.2.3 (+https://foo.com/twtxt.txt; @foo)`,
err: nil,
expected: &TwtxtUserAgent{
Client: "twtxt/1.2.3",
name: "Single-User Twtxt User Agent",
ua: `twtxt/1.2.3 (+https://foo.com/twtxt.txt; @foo)`,
expected: &SingleUserAgent{
twtxtUserAgent: twtxtUserAgent{Client: "twtxt/1.2.3"},
Nick: "foo",
URL: "https://foo.com/twtxt.txt",
},
},
{
name: "Multi-User Twtxt User Agent",
ua: `yarnd/0.8.0@d4e265e (~https://example.com/whoFollows?followers=14&token=iABA0yhUz; contact=https://example.com/support)`,
expected: &MultiUserAgent{
twtxtUserAgent: twtxtUserAgent{Client: "yarnd/0.8.0@d4e265e"},
WhoFollowsURL: "https://example.com/whoFollows?followers=14&token=iABA0yhUz",
SupportURL: "https://example.com/support",
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
actual, err := ParseUserAgent(testCase.ua)
if testCase.err != nil {
assert.Equal(t, testCase.err, err)
} else {
assert.NoError(t, err)
assert.IsType(t, testCase.expected, actual)
switch act := actual.(type) {
case *SingleUserAgent:
assert.Equal(t, testCase.expected.(*SingleUserAgent).Client, act.Client)
assert.Equal(t, testCase.expected.(*SingleUserAgent).Nick, act.Nick)
assert.Equal(t, testCase.expected.(*SingleUserAgent).URL, act.URL)
case *MultiUserAgent:
assert.Equal(t, testCase.expected.(*MultiUserAgent).Client, act.Client)
assert.Equal(t, testCase.expected.(*MultiUserAgent).WhoFollowsURL, act.WhoFollowsURL)
assert.Equal(t, testCase.expected.(*MultiUserAgent).SupportURL, act.SupportURL)
default:
assert.Fail(t, "test setup error: unsupported user agent type")
}
}
})
}
}
func TestTwtxtUserAgent_IsPod(t *testing.T) {
testCases := []struct{
name string
ua string
expected bool
}{
{
name: "Single-User non-yarnd User Agent",
ua: "twtxt/1.2.3 (+https://example.com/twtxt.txt; @foo)",
expected: false,
},
{
name: "Single-User yarnd User Agent",
ua: "yarnd/0.8.0@d4e265e (+https://example.com/user/foo/twtxt.txt; @foo)",
expected: true,
},
{
name: "Multi-User non-yarnd User Agent",
ua: "bernd/0.8.0@d4e265e (~https://example.com/whoFollows?followers=14&token=iABA0yhUz; contact=https://example.com/support)",
expected: false,
},
{
name: "Multi-User yarnd User Agent",
ua: "yarnd/0.8.0@d4e265e (~https://example.com/whoFollows?followers=14&token=iABA0yhUz; contact=https://example.com/support)",
expected: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
ua, err := ParseUserAgent(testCase.ua)
assert.NoError(t, err)
assert.Equal(t, testCase.expected, ua.IsPod())
})
}
}
func TestTwtxtUserAgent_PodBaseURL(t *testing.T) {
testCases := []struct{
name string
ua string
expected string
}{
{
name: "Single-User non-yarnd User Agent",
ua: "twtxt/1.2.3 (+https://example.com/twtxt.txt; @foo)",
expected: "",
},
{
name: "Single-User yarnd User Agent",
ua: "yarnd/0.8.0@d4e265e (+https://example.com/user/foo/twtxt.txt; @foo)",
expected: "https://example.com",
},
{
name: "Single-User yarnd with subdirectory User Agent",
ua: "yarnd/0.8.0@d4e265e (+https://example.com/subdir/user/foo/twtxt.txt; @foo)",
expected: "https://example.com/subdir",
},
{
name: "Multi-User non-yarnd User Agent",
ua: "bernd/0.8.0@d4e265e (~https://example.com/whoFollows?followers=14&token=iABA0yhUz; contact=https://example.com/support)",
expected: "",
},
{
name: "Multi-User yarnd User Agent",
ua: "yarnd/0.8.0@d4e265e (~https://example.com/whoFollows?followers=14&token=iABA0yhUz; contact=https://example.com/support)",
expected: "https://example.com",
},
{
name: "Multi-User yarnd with subdirectory User Agent",
ua: "yarnd/0.8.0@d4e265e (~https://example.com/subdir/whoFollows?followers=14&token=iABA0yhUz; contact=https://example.com/subdir/support)",
expected: "https://example.com/subdir",
},
}
for _, testCase := range testCases {
actual, err := ParseTwtxtUserAgent(testCase.ua)
if err != nil {
assert.Equal(t, testCase.err, err)
} else {