Browse Source

Bypass Cache.ShouldRefreshFeed() on FetchFeedRequest.Force

James Mills 1 week ago
parent
commit
707b211e84
Signed by: prologic GPG Key ID: AC4C014F1440EBD6
  1. 14
      internal/api.go
  2. 13
      internal/cache.go
  3. 8
      internal/external_handlers.go
  4. 11
      internal/interpod_protocol.go
  5. 10
      internal/jobs.go
  6. 16
      internal/models.go
  7. 2
      internal/utils.go
  8. 16
      types/feed.go
  9. 2
      types/feed_test.go

14
internal/api.go

@ -388,7 +388,7 @@ func (a *API) PostEndpoint() httprouter.Handle {
return
}
var sources types.Feeds
var sources types.FetchFeedRequests
switch req.PostAs {
case "", me:
@ -982,8 +982,8 @@ func (a *API) ProfileEndpoint() httprouter.Handle {
}
if !a.cache.IsCached(profile.URI) {
sources := make(types.Feeds)
sources[types.Feed{Nick: profile.Nick, URL: profile.URI}] = true
sources := make(types.FetchFeedRequests)
sources[types.FetchFeedRequest{Nick: profile.Nick, URL: profile.URI}] = true
a.cache.FetchFeeds(a.config, a.archive, sources, nil)
}
@ -1114,8 +1114,8 @@ func (a *API) FetchTwtsEndpoint() httprouter.Handle {
if req.URL != "" && !isLocal(req.URL) {
if !a.cache.IsCached(req.URL) {
sources := make(types.Feeds)
sources[types.Feed{Nick: nick, URL: req.URL}] = true
sources := make(types.FetchFeedRequests)
sources[types.FetchFeedRequest{Nick: nick, URL: req.URL}] = true
a.cache.FetchFeeds(a.config, a.archive, sources, nil)
}
@ -1196,8 +1196,8 @@ func (a *API) ExternalProfileEndpoint() httprouter.Handle {
if !a.cache.IsCached(uri) {
a.tasks.DispatchFunc(func() error {
sources := make(types.Feeds)
sources[types.Feed{Nick: nick, URL: uri}] = true
sources := make(types.FetchFeedRequests)
sources[types.FetchFeedRequest{Nick: nick, URL: uri}] = true
a.cache.FetchFeeds(a.config, a.archive, sources, nil)
return nil
})

13
internal/cache.go

@ -850,7 +850,7 @@ func (cache *Cache) DetectPodFromUserAgent(ua TwtxtUserAgent) error {
}
// FetchFeeds ...
func (cache *Cache) FetchFeeds(conf *Config, archive Archiver, feeds types.Feeds, publicFollowers map[types.Feed][]string) {
func (cache *Cache) FetchFeeds(conf *Config, archive Archiver, feeds types.FetchFeedRequests, publicFollowers map[types.FetchFeedRequest][]string) {
stime := time.Now()
defer func() {
metrics.Gauge(
@ -893,7 +893,7 @@ func (cache *Cache) FetchFeeds(conf *Config, archive Archiver, feeds types.Feeds
fetchers <- struct{}{}
// anon func takes needed variables as arg, avoiding capture of iterator variables
go func(feed types.Feed) {
go func(feed types.FetchFeedRequest) {
defer func() {
<-fetchers
wg.Done()
@ -914,11 +914,12 @@ func (cache *Cache) FetchFeeds(conf *Config, archive Archiver, feeds types.Feeds
}
// Handle Feed Refresh
// Supports two methods of refresh:
// Supports three methods of refresh:
// 1) A refresh interval (suggested refresh interval by feed author), e.g:
// # refresh = 1h
// 2) An exponential back-off based on a weighted moving average of a feed's update frequency (TBD)
if !cache.ShouldRefreshFeed(feed.URL) {
// 3) FetchFeedRequest.Force is `true` so we fetch the feed immediately (Subscription Notification)
if !feed.Force && !cache.ShouldRefreshFeed(feed.URL) {
twtsch <- nil
return
}
@ -1416,7 +1417,7 @@ func (cache *Cache) ShouldRefreshFeed(uri string) bool {
}
// Always refresh feeds on the same pod.
if IsLocalURLFactory(cache.conf)(uri) {
if cache.conf.IsLocalURL(uri) {
return true
}
@ -1758,7 +1759,7 @@ func (cache *Cache) DeleteUserViews(u *User) {
}
// DeleteFeeds ...
func (cache *Cache) DeleteFeeds(feeds types.Feeds) {
func (cache *Cache) DeleteFeeds(feeds types.FetchFeedRequests) {
cache.mu.Lock()
for feed := range feeds {
delete(cache.Feeds, feed.URL)

8
internal/external_handlers.go

@ -37,8 +37,8 @@ func (s *Server) ExternalHandler() httprouter.Handle {
if !s.cache.IsCached(uri) {
s.tasks.DispatchFunc(func() error {
sources := make(types.Feeds)
sources[types.Feed{Nick: nick, URL: uri}] = true
sources := make(types.FetchFeedRequests)
sources[types.FetchFeedRequest{Nick: nick, URL: uri}] = true
s.cache.FetchFeeds(s.config, s.archive, sources, nil)
return nil
})
@ -160,8 +160,8 @@ func (s *Server) ExternalFollowingHandler() httprouter.Handle {
if !s.cache.IsCached(uri) {
s.tasks.DispatchFunc(func() error {
sources := make(types.Feeds)
sources[types.Feed{Nick: nick, URL: uri}] = true
sources := make(types.FetchFeedRequests)
sources[types.FetchFeedRequest{Nick: nick, URL: uri}] = true
s.cache.FetchFeeds(s.config, s.archive, sources, nil)
return nil
})

11
internal/interpod_protocol.go

@ -207,11 +207,12 @@ func (s *Server) IPPPubHandler() httprouter.Handle {
// Refresh the feed.
s.tasks.DispatchFunc(func() error {
sources := make(types.Feeds)
publicFollowers := make(map[types.Feed][]string)
feed := types.Feed{
Nick: twter.Nick,
URL: twter.URI,
sources := make(types.FetchFeedRequests)
publicFollowers := make(map[types.FetchFeedRequest][]string)
feed := types.FetchFeedRequest{
Nick: twter.Nick,
URL: twter.URI,
Force: true,
}
sources[feed] = true
users, err := s.db.GetAllUsers()

10
internal/jobs.go

@ -204,22 +204,22 @@ func (job *UpdateFeedsJob) Run() {
log.Infof("updating feeds for %d users and %d feeds", len(users), len(feeds))
sources := make(types.Feeds)
publicFollowers := make(map[types.Feed][]string)
sources := make(types.FetchFeedRequests)
publicFollowers := make(map[types.FetchFeedRequest][]string)
// Ensure all specialUsername feeds are in the cache
for _, username := range specialUsernames {
sources[types.Feed{Nick: username, URL: URLForUser(job.conf.BaseURL, username)}] = true
sources[types.FetchFeedRequest{Nick: username, URL: URLForUser(job.conf.BaseURL, username)}] = true
}
// Ensure all twtxtBots feeds are in the cache
for _, bot := range automatedFeeds {
sources[types.Feed{Nick: bot, URL: URLForUser(job.conf.BaseURL, bot)}] = true
sources[types.FetchFeedRequest{Nick: bot, URL: URLForUser(job.conf.BaseURL, bot)}] = true
}
for _, feed := range feeds {
// Ensure we fetch the feed's own posts in the cache
sources[types.Feed{Nick: feed.Name, URL: feed.URL}] = true
sources[types.FetchFeedRequest{Nick: feed.Name, URL: feed.URL}] = true
}
for _, user := range users {

16
internal/models.go

@ -255,9 +255,9 @@ func (f *Feed) FollowedBy(url string) bool {
return ok
}
func (f *Feed) Source() types.Feeds {
feeds := make(types.Feeds)
feeds[types.Feed{Nick: f.Name, URL: f.URL}] = true
func (f *Feed) Source() types.FetchFeedRequests {
feeds := make(types.FetchFeedRequests)
feeds[types.FetchFeedRequest{Nick: f.Name, URL: f.URL}] = true
return feeds
}
@ -464,17 +464,17 @@ func (u *User) HasMuted(url string) bool {
return ok
}
func (u *User) Source() types.Feeds {
feeds := make(types.Feeds)
feeds[types.Feed{Nick: u.Username, URL: u.URL}] = true
func (u *User) Source() types.FetchFeedRequests {
feeds := make(types.FetchFeedRequests)
feeds[types.FetchFeedRequest{Nick: u.Username, URL: u.URL}] = true
return feeds
}
func (u *User) Sources() types.Feeds {
func (u *User) Sources() types.FetchFeedRequests {
// Ensure we fetch the user's own posts in the cache
feeds := u.Source()
for url, nick := range u.sources {
feeds[types.Feed{Nick: nick, URL: url}] = true
feeds[types.FetchFeedRequest{Nick: nick, URL: url}] = true
}
return feeds
}

2
internal/utils.go

@ -1586,7 +1586,7 @@ func URLForTask(baseURL, uuid string) string {
)
}
func URLForWhoFollows(baseURL string, feed types.Feed, feedFollowers int) string {
func URLForWhoFollows(baseURL string, feed types.FetchFeedRequest, feedFollowers int) string {
return fmt.Sprintf(
"%s/whoFollows?followers=%d&token=%s",
strings.TrimSuffix(baseURL, "/"),

16
types/feed.go

@ -2,17 +2,21 @@ package types
import "fmt"
// Feed is an single twtxt.txt feed with a cannonical Nickname and URL for the feed
type Feed struct {
// FetchFeedRequest is an single request for a twtxt.txt feed with a cannonical Nickname and URL for the feed
// and optinoal request parameters that affect how the Cache fetches the feed.
type FetchFeedRequest struct {
Nick string
URL string
// Force whether or not to immediately fetch the feed and bypass Cache.ShouldRefreshFeed()
Force bool
}
// String implements the Stringer interface and returns the Feed represented
// as a twtxt.txt URI in the form @<nick url>
func (f Feed) String() string {
return fmt.Sprintf("@<%s %s>", f.Nick, f.URL)
func (f FetchFeedRequest) String() string {
return fmt.Sprintf("FetchFeedRequest: @<%s %s>", f.Nick, f.URL)
}
// Feeds is a mappping of Feed to booleans used to ensure unique feeds
type Feeds map[Feed]bool
// FetchFeedRequests is a mappping of FetchFeedRequest to booleans used to ensure unique feeds
type FetchFeedRequests map[FetchFeedRequest]bool

2
types/feed_test.go

@ -10,7 +10,7 @@ func TestFeed(t *testing.T) {
assert := assert.New(t)
t.Run("String", func(t *testing.T) {
f := Feed{Nick: "prologic", URL: "https://twtxt.net/user/prologic/twtxt.txt"}
f := FetchFeedRequest{Nick: "prologic", URL: "https://twtxt.net/user/prologic/twtxt.txt"}
assert.Equal("@<prologic https://twtxt.net/user/prologic/twtxt.txt>", f.String())
})
}

Loading…
Cancel
Save