Fix feed rotation edge case by introducing a -S/--max-feed-size option

pull/1080/head
James Mills 2 weeks ago
parent 7d4244d04a
commit d853649020
Signed by: prologic
GPG Key ID: AC4C014F1440EBD6
  1. 6
      cmd/yarnd/main.go
  2. 1
      internal/config.go
  3. 4
      internal/jobs.go
  4. 11
      internal/options.go
  5. 1
      internal/server.go

@ -99,6 +99,7 @@ var (
maxTwtLength int
maxUploadSize int64
maxFetchLimit int64
maxFeedSize int64
maxCacheFetchers int
maxCacheTTL time.Duration
fetchInterval string
@ -207,6 +208,10 @@ func initFlags() {
&maxFetchLimit, "max-fetch-limit", "F", internal.DefaultMaxFetchLimit,
"maximum feed fetch limit in bytes",
)
flag.Int64VarP(
&maxFeedSize, "max-feed-size", "S", internal.DefaultMaxFeedSize,
"maximum feed size before a feed is rorated inbytes",
)
flag.DurationVarP(
&maxCacheTTL, "max-cache-ttl", "C", internal.DefaultMaxCacheTTL,
"maximum cache ttl (time-to-live) of cached twts in memory",
@ -385,6 +390,7 @@ func main() {
internal.WithMaxTwtLength(maxTwtLength),
internal.WithMaxUploadSize(maxUploadSize),
internal.WithMaxFetchLimit(maxFetchLimit),
internal.WithMaxFeedSize(maxFeedSize),
internal.WithMaxCacheFetchers(maxCacheFetchers),
internal.WithMaxCacheTTL(maxCacheTTL),
internal.WithFetchInterval(fetchInterval),

@ -158,6 +158,7 @@ type Config struct {
MaxCacheFetchers int
MaxFetchLimit int64
MaxFeedSize int64
APISessionTime time.Duration `json:"-"`
APISigningKey string `json:"-"`

@ -335,8 +335,8 @@ func (job *RotateFeedsJob) Run() {
continue
}
if stat.Size() > job.conf.MaxFetchLimit {
log.Infof("rotating %s with size %s > %s", feed, humanize.Bytes(uint64(stat.Size())), humanize.Bytes(uint64(job.conf.MaxFetchLimit)))
if stat.Size() > job.conf.MaxFeedSize {
log.Infof("rotating %s with size %s > %s", feed, humanize.Bytes(uint64(stat.Size())), humanize.Bytes(uint64(job.conf.MaxFeedSize)))
if err := RotateFeed(job.conf, feed); err != nil {
log.WithError(err).Error("error rotating feed")

@ -145,6 +145,9 @@ const (
// DefaultMaxFetchLimit is the maximum fetch fetch limit in bytes
DefaultMaxFetchLimit = 1 << 20 // ~1MB (or more than enough for months)
// DefaultMaxFeedSize is the maximum feed size before a feed is rotated (should be smaller than MaxFetchLimit)
DefaultMaxFeedSize = DefaultMaxFetchLimit / 2
// DefaultAPISessionTime is the server's default session time for API tokens
DefaultAPISessionTime = 240 * time.Hour // 10 days
@ -630,6 +633,14 @@ func WithMaxFetchLimit(limit int64) Option {
}
}
// WithMaxFeedSize sets the maximum feed size before a feed is rotated
func WithMaxFeedSize(size int64) Option {
return func(cfg *Config) error {
cfg.MaxFeedSize = size
return nil
}
}
// WithAPISessionTime sets the API session time for tokens
func WithAPISessionTime(duration time.Duration) Option {
return func(cfg *Config) error {

@ -1173,6 +1173,7 @@ func NewServer(bind string, options ...Option) (*Server, error) {
log.Infof("SMTP User: %s", server.config.SMTPUser)
log.Infof("SMTP From: %s", server.config.SMTPFrom)
log.Infof("Max Fetch Limit: %s", humanize.Bytes(uint64(server.config.MaxFetchLimit)))
log.Infof("Max Feed Size: %s", humanize.Bytes(uint64(server.config.MaxFeedSize)))
log.Infof("Max Upload Size: %s", humanize.Bytes(uint64(server.config.MaxUploadSize)))
log.Infof("API Session Time: %s", server.config.APISessionTime)
log.Infof("Enabled Features: %s", server.config.Features)

Loading…
Cancel
Save