Major refactor

pull/14/head
James Mills 2 years ago
parent 914268a320
commit 2d9b59beb0
Signed by: prologic
GPG Key ID: AC4C014F1440EBD6

4
.gitignore vendored

@ -3,6 +3,6 @@
*.txt
.DS_Store
/feeds
/data
/rss2twtxt
/config.yaml
/feeds.yaml

@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${fileDirname}",
"args": ["-s"]
}
],
}

@ -14,25 +14,27 @@ import (
)
type App struct {
bind string
conf *Config
cron *cron.Cron
router *mux.Router
conf *Config
cron *cron.Cron
}
func NewApp(bind, config string) (*App, error) {
conf, err := LoadConfig(config)
if err != nil {
return nil, err
func NewApp(options ...Option) (*App, error) {
conf := NewConfig()
for _, opt := range options {
if err := opt(conf); err != nil {
return nil, err
}
}
if err := conf.LoadFeeds(); err != nil {
log.WithError(err).Error("error loading feeds")
return nil, fmt.Errorf("error loading feeds: %w", err)
}
cron := cron.New()
return &App{
bind: bind,
conf: conf,
cron: cron,
}, nil
return &App{conf: conf, cron: cron}, nil
}
func (app *App) initRoutes() *mux.Router {
@ -76,7 +78,7 @@ func (app *App) runStartupJobs() {
}
func (app *App) GetFeeds() (feeds []Feed) {
files, err := WalkMatch(app.conf.Root, "*.txt")
files, err := WalkMatch(app.conf.DataDir, "*.txt")
if err != nil {
log.WithError(err).Error("error reading feeds directory")
return nil
@ -92,8 +94,8 @@ func (app *App) GetFeeds() (feeds []Feed) {
}
lastModified := humanize.Time(stat.ModTime())
url := fmt.Sprintf("%s/%s/twtxt.txt", app.conf.BaseURL, name)
feeds = append(feeds, Feed{name, url, lastModified})
uri := fmt.Sprintf("%s/%s/twtxt.txt", app.conf.BaseURL, name)
feeds = append(feeds, Feed{Name: name, URI: uri, LastModified: lastModified})
}
sort.Slice(feeds, func(i, j int) bool { return feeds[i].Name < feeds[j].Name })
@ -111,9 +113,9 @@ func (app *App) Run() error {
app.cron.Start()
log.Info("started background jobs")
log.Infof("rss2twtxt %s listening on http://%s", FullVersion(), app.bind)
log.Infof("feeds %s listening on http://%s", FullVersion(), app.conf.Addr)
go app.runStartupJobs()
return http.ListenAndServe(app.bind, router)
return http.ListenAndServe(app.conf.Addr, router)
}

@ -1,47 +1,78 @@
package main
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/go-yaml/yaml"
log "github.com/sirupsen/logrus"
)
type Config struct {
Root string
BaseURL string
MaxSize int64 // maximum feed size before rotating
Feeds map[string]string // name -> url
Addr string
Debug bool
path string // path to config file that was loaded used by .Save()
}
DataDir string
BaseURL string
FeedsFile string
MaxFeedSize int64 // maximum feed size before rotating
func (conf *Config) Parse(data []byte) error {
return yaml.Unmarshal(data, conf)
Feeds map[string]*Feed // name -> url
}
func (conf *Config) Save() error {
data, err := yaml.Marshal(conf)
func (conf *Config) LoadFeeds() error {
f, err := os.Open(conf.FeedsFile)
if err != nil {
return err
log.WithError(err).Errorf("error opening feeds file %s", conf.FeedsFile)
return fmt.Errorf("error opening feeds file %s: %w", conf.FeedsFile, err)
}
data = append([]byte("---\n"), data...)
return ioutil.WriteFile(conf.path, data, 0644)
defer f.Close()
data, err := ioutil.ReadAll(f)
if err != nil {
log.WithError(err).Errorf("error reading feeds file %s", conf.FeedsFile)
return fmt.Errorf("error reading feeds file %s: %w", conf.FeedsFile, err)
}
if err := yaml.Unmarshal(data, conf.Feeds); err != nil {
log.WithError(err).Errorf("error parsing feeds file %s", conf.FeedsFile)
return fmt.Errorf("error parsing feeds file %s: %w", conf.FeedsFile, err)
}
for _, feed := range conf.Feeds {
fn := filepath.Join(conf.DataDir, fmt.Sprintf("%s.png", feed.Name))
log.Debugf("Exists(fn): %t", Exists(fn))
log.Debugf("feed.Avatar: %q", feed.Avatar)
if Exists(fn) && feed.Avatar == "" {
feed.Avatar = fmt.Sprintf("%s/%s/avatar.png", conf.BaseURL, feed.Name)
}
}
return nil
}
func LoadConfig(filename string) (*Config, error) {
data, err := ioutil.ReadFile(filename)
func (conf *Config) SaveFeeds() error {
f, err := os.OpenFile(conf.FeedsFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, err
log.WithError(err).Errorf("error opening feeds file %s", conf.FeedsFile)
return fmt.Errorf("error opening feeds file %s: %w", conf.FeedsFile, err)
}
conf := &Config{}
if err := conf.Parse(data); err != nil {
return nil, err
defer f.Close()
data, err := yaml.Marshal(conf.Feeds)
if err != nil {
log.WithError(err).Errorf("error serializing feeds")
return fmt.Errorf("error serializing feeds: %w", err)
}
conf.path = filename
if conf.Feeds == nil {
conf.Feeds = make(map[string]string)
data = append([]byte("---\n"), data...)
if _, err := f.Write(data); err != nil {
log.WithError(err).Errorf("error writing feeds file %s", conf.FeedsFile)
return fmt.Errorf("error writing feeds file %s: %w", conf.FeedsFile, err)
}
return conf, nil
return nil
}

@ -1,6 +0,0 @@
---
root: ./feeds
baseurl: /
maxsize: 1048576
feeds:
unexplained_mysteries: http://www.unexplained-mysteries.com/news/umnews.xml

@ -35,6 +35,9 @@ type Feed struct {
Name string
URI string
Avatar string
Description string
LastModified string
}
@ -82,6 +85,16 @@ func TestRSSFeed(uri string) (*gofeed.Feed, error) {
return feed, nil
}
func FindRSSOrAtomAlternate(alts []*microformats.AlternateRel) string {
for _, alt := range alts {
switch alt.Type {
case "application/atom+xml", "application/rss+xml":
return alt.URL
}
}
return ""
}
func FindRSSFeed(uri string) (*gofeed.Feed, string, error) {
u, err := url.Parse(uri)
if err != nil {
@ -105,13 +118,7 @@ func FindRSSFeed(uri string) (*gofeed.Feed, string, error) {
feedURI := altMap["application/atom+xml"]
if feedURI == "" {
for _, alt := range data.Alternates {
switch alt.Type {
case "application/atom+xml", "application/rss+xml":
feedURI = alt.URL
break
}
}
feedURI = FindRSSOrAtomAlternate(data.Alternates)
}
if feedURI == "" {
@ -178,7 +185,7 @@ func ValidateRSSFeed(conf *Config, uri string) (Feed, error) {
func UpdateTwitterFeed(conf *Config, name, handle string) error {
var lastModified = time.Time{}
fn := filepath.Join(conf.Root, fmt.Sprintf("%s.txt", name))
fn := filepath.Join(conf.DataDir, fmt.Sprintf("%s.txt", name))
stat, err := os.Stat(fn)
if err == nil {
@ -247,7 +254,7 @@ func UpdateRSSFeed(conf *Config, name, url string) error {
return err
}
avatarFile := filepath.Join(conf.Root, fmt.Sprintf("%s.png", name))
avatarFile := filepath.Join(conf.DataDir, fmt.Sprintf("%s.png", name))
if feed.Image != nil && feed.Image.URL != "" && !Exists(avatarFile) {
opts := &ImageOptions{
Resize: true,
@ -264,7 +271,7 @@ func UpdateRSSFeed(conf *Config, name, url string) error {
var lastModified = time.Time{}
fn := filepath.Join(conf.Root, fmt.Sprintf("%s.txt", name))
fn := filepath.Join(conf.DataDir, fmt.Sprintf("%s.txt", name))
stat, err := os.Stat(fn)
if err == nil {

@ -45,7 +45,7 @@ func renderMessage(w http.ResponseWriter, status int, title, message string) err
func (app *App) HealthHandler(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
return
http.Error(w, "Healthy", http.StatusOK)
}
func (app *App) IndexHandler(w http.ResponseWriter, r *http.Request) {
@ -118,8 +118,8 @@ func (app *App) IndexHandler(w http.ResponseWriter, r *http.Request) {
return
}
app.conf.Feeds[feed.Name] = feed.URI
if err := app.conf.Save(); err != nil {
app.conf.Feeds[feed.Name] = &feed
if err := app.conf.SaveFeeds(); err != nil {
msg := fmt.Sprintf("Could not save feed: %s", err)
if err := renderMessage(w, http.StatusInternalServerError, "Error", msg); err != nil {
log.WithError(err).Error("error rendering message template")
@ -150,14 +150,21 @@ func (app *App) FeedHandler(w http.ResponseWriter, r *http.Request) {
return
}
filename := filepath.Join(app.conf.Root, fmt.Sprintf("%s.txt", name))
if !Exists(filename) {
fn := filepath.Join(app.conf.DataDir, fmt.Sprintf("%s.txt", name))
if !Exists(fn) {
log.Warnf("feed does not exist %s", name)
http.Error(w, "Feed not found", http.StatusNotFound)
return
}
fileInfo, err := os.Stat(filename)
feed, ok := app.conf.Feeds[name]
if !ok {
log.Warnf("feed does not exist %s", name)
http.Error(w, "Feed not found", http.StatusNotFound)
return
}
fileInfo, err := os.Stat(fn)
if err != nil {
log.WithError(err).Error("os.Stat() error")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@ -165,12 +172,47 @@ func (app *App) FeedHandler(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprintf("%d", fileInfo.Size()))
w.Header().Set("Last-Modified", fileInfo.ModTime().Format(http.TimeFormat))
if r.Method == http.MethodHead {
return
}
http.ServeFile(w, r, filename)
ctx := map[string]string{
"Name": feed.Name,
"URL": fmt.Sprintf("%s/%s/twtxt.txt", app.conf.BaseURL, feed.Name),
"Source": feed.URI,
"Avatar": feed.Avatar,
"Description": feed.Description,
"LastModified": fileInfo.ModTime().UTC().Format(time.RFC3339),
}
preamble, err := RenderPlainText(preambleTemplate, ctx)
if err != nil {
log.WithError(err).Warn("error rendering twtxt preamble")
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprintf("%d", int64(len(preamble))+fileInfo.Size()))
w.Header().Set("Last-Modified", fileInfo.ModTime().UTC().Format(http.TimeFormat))
if r.Method == http.MethodHead {
return
}
if _, err = w.Write([]byte(preamble)); err != nil {
log.WithError(err).Warn("error writing twtxt preamble")
}
f, err := os.Open(fn)
if err != nil {
log.WithError(err).Error("error opening feed")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
defer f.Close()
_, _ = io.Copy(w, f)
return
}
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
@ -189,14 +231,14 @@ func (app *App) AvatarHandler(w http.ResponseWriter, r *http.Request) {
return
}
filename := filepath.Join(app.conf.Root, fmt.Sprintf("%s.txt", name))
if !Exists(filename) {
fn := filepath.Join(app.conf.DataDir, fmt.Sprintf("%s.txt", name))
if !Exists(fn) {
log.Warnf("feed does not exist %s", name)
http.Error(w, "Feed not found", http.StatusNotFound)
return
}
fn := filepath.Join(app.conf.Root, fmt.Sprintf("%s.png", name))
fn = filepath.Join(app.conf.DataDir, fmt.Sprintf("%s.png", name))
if fileInfo, err := os.Stat(fn); err == nil {
etag := fmt.Sprintf("W/\"%s-%s\"", r.RequestURI, fileInfo.ModTime().Format(time.RFC3339))

@ -52,7 +52,7 @@ func NewRotateFeedsJob(conf *Config) cron.Job {
func (job *RotateFeedsJob) Run() {
conf := job.conf
files, err := WalkMatch(conf.Root, "*.txt")
files, err := WalkMatch(conf.DataDir, "*.txt")
if err != nil {
log.WithError(err).Error("error reading feeds directory")
return
@ -65,12 +65,12 @@ func (job *RotateFeedsJob) Run() {
continue
}
if stat.Size() > conf.MaxSize {
if stat.Size() > conf.MaxFeedSize {
log.Infof(
"rotating %s with size %s > %s",
BaseWithoutExt(file),
humanize.Bytes(uint64(stat.Size())),
humanize.Bytes(uint64(conf.MaxSize)),
humanize.Bytes(uint64(conf.MaxFeedSize)),
)
if err := RotateFile(file); err != nil {
@ -90,22 +90,22 @@ func NewUpdateFeedsJob(conf *Config) cron.Job {
func (job *UpdateFeedsJob) Run() {
conf := job.conf
for name, uri := range conf.Feeds {
u, err := ParseURI(uri)
for name, feed := range conf.Feeds {
u, err := ParseURI(feed.URI)
if err != nil {
log.WithError(err).Errorf("error parsing feed %s: %s", name, uri)
log.WithError(err).Errorf("error parsing feed %s: %s", name, feed.URI)
} else {
switch u.Type {
case "rss", "http", "https":
if err := UpdateRSSFeed(conf, name, uri); err != nil {
log.WithError(err).Errorf("error updating rss feed %s: %s", name, uri)
if err := UpdateRSSFeed(conf, name, feed.URI); err != nil {
log.WithError(err).Errorf("error updating rss feed %s: %s", name, feed.URI)
}
case "twitter":
if err := UpdateTwitterFeed(conf, name, u.Config); err != nil {
log.WithError(err).Errorf("error updating twitter feed %s: %s", name, uri)
log.WithError(err).Errorf("error updating twitter feed %s: %s", name, feed.URI)
}
default:
log.Warnf("error unknown feed type %s: %s", name, uri)
log.Warnf("error unknown feed type %s: %s", name, feed.URI)
}
}
}
@ -138,6 +138,14 @@ func NewTikTokJob(conf *Config) cron.Job {
name := "tiktok"
url := fmt.Sprintf("@<%s %s>", name, URLForFeed(conf, name))
conf.Feeds[name] = &Feed{
Name: name,
Description: fmt.Sprintf(
"I am @%s an automated feed that twts every 30m with the current time (UTC)",
name,
),
}
return &TikTokJob{
conf: conf,
name: name,
@ -149,7 +157,7 @@ func NewTikTokJob(conf *Config) cron.Job {
func (job *TikTokJob) Run() {
conf := job.conf
fn := filepath.Join(conf.Root, fmt.Sprintf("%s.txt", job.name))
fn := filepath.Join(conf.DataDir, fmt.Sprintf("%s.txt", job.name))
f, err := os.OpenFile(fn, os.O_APPEND|os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
@ -158,18 +166,6 @@ func (job *TikTokJob) Run() {
}
defer f.Close()
err = AppendTwt(f,
fmt.Sprintf(`
I am %s an automated feed that twts every 30m with the current time (UTC)
`, job.url,
),
time.Unix(0, 0),
)
if err != nil {
log.WithError(err).Error("error writing @tiktok feed")
return
}
now := time.Now().UTC()
hour := now.Hour() % 12

@ -3,6 +3,7 @@ package main
import (
"fmt"
"os"
"strings"
log "github.com/sirupsen/logrus"
flag "github.com/spf13/pflag"
@ -12,9 +13,11 @@ var (
version bool
debug bool
server bool
bind string
config string
bind string
server bool
baseURL string
dataDir string
feedsFile string
)
func init() {
@ -24,15 +27,44 @@ func init() {
}
flag.BoolVarP(&version, "version", "v", false, "display version information")
flag.BoolVarP(&debug, "debug", "d", false, "enable debug logging")
flag.BoolVarP(&debug, "debug", "D", false, "enable debug logging")
flag.BoolVarP(&server, "server", "s", false, "enable server mode")
flag.StringVarP(&bind, "bind", "b", "0.0.0.0:8000", "interface and port to bind to in server mode")
flag.StringVarP(&config, "config", "c", "config.yaml", "configuration file for server mode")
flag.BoolVarP(&server, "server", "s", false, "enable server mode")
flag.StringVarP(&dataDir, "data-dir", "d", "./data", "data directory to store feeds in")
flag.StringVarP(&baseURL, "base-url", "u", "http://0.0.0.0:8000", "base url for generated urls")
flag.StringVarP(&feedsFile, "feeds-file", "f", "feeds.yaml", "feeds configuration file in server mode")
}
func main() {
func flagNameFromEnvironmentName(s string) string {
s = strings.ToLower(s)
s = strings.Replace(s, "_", "-", -1)
return s
}
func parseArgs() error {
for _, v := range os.Environ() {
vals := strings.SplitN(v, "=", 2)
flagName := flagNameFromEnvironmentName(vals[0])
fn := flag.CommandLine.Lookup(flagName)
if fn == nil || fn.Changed {
continue
}
if err := fn.Value.Set(vals[1]); err != nil {
return err
}
}
flag.Parse()
return nil
}
func main() {
parseArgs()
if version {
fmt.Printf("feeds %s\n", FullVersion())
os.Exit(0)
}
if debug {
log.SetLevel(log.DebugLevel)
@ -40,13 +72,13 @@ func main() {
log.SetLevel(log.InfoLevel)
}
if version {
fmt.Printf("rss2twtxt %s\n", FullVersion())
os.Exit(0)
}
if server {
app, err := NewApp(bind, config)
app, err := NewApp(
WithBind(bind),
WithDataDir(dataDir),
WithBaseURL(baseURL),
WithFeedsFile(feedsFile),
)
if err != nil {
log.WithError(err).Fatal("error creating app for server mode")
}
@ -59,7 +91,7 @@ func main() {
uri := flag.Arg(0)
name := flag.Arg(1)
conf := &Config{Root: "."}
conf := &Config{DataDir: "."}
u, err := ParseURI(uri)
if err != nil {

@ -0,0 +1,86 @@
package main
import (
"fmt"
"os"
)
const (
// DefaultDebug is the default debug mode
DefaultDebug = false
// DefaultDataDir is the default data directory for storage
DefaultDataDir = "./data"
// DefaultAddr is the default bind address of the server
DefaultAddr = "0.0.0.0:8000"
// DefaultBaseURL is the default Base URL for the app used to construct feed URLs
DefaultBaseURL = "http://0.0.0.0:8000"
// DefaultFeedsFile is the default feeds configuration filename used by the server
DefaultFeedsFile = "feeds.yaml"
// DefaultMaxFeedSize is the default maximum feed size before rotation
DefaultMaxFeedSize = 1 << 20 // ~1MB
)
func NewConfig() *Config {
return &Config{
Addr: DefaultAddr,
Debug: DefaultDebug,
DataDir: DefaultDataDir,
BaseURL: DefaultBaseURL,
FeedsFile: DefaultFeedsFile,
MaxFeedSize: DefaultMaxFeedSize,
Feeds: make(map[string]*Feed),
}
}
// Option is a function that takes a config struct and modifies it
type Option func(*Config) error
// WithDebug sets the debug mode lfag
func WithDebug(debug bool) Option {
return func(cfg *Config) error {
cfg.Debug = debug
return nil
}
}
// WithBind sets the server's listening bind address
func WithBind(addr string) Option {
return func(cfg *Config) error {
cfg.Addr = addr
return nil
}
}
// WithDataDir sets the data directory to use for storage
func WithDataDir(dataDir string) Option {
return func(cfg *Config) error {
if err := os.MkdirAll(dataDir, 0755); err != nil {
return fmt.Errorf("error creating data-dir %s: %w", dataDir, err)
}
cfg.DataDir = dataDir
return nil
}
}
// WithFeedsFile set the feeds configuration file used by the server
func WithFeedsFile(feedsFile string) Option {
return func(cfg *Config) error {
cfg.FeedsFile = feedsFile
return nil
}
}
// WithBaseURL sets the Base URL used for constructing feed URLs
func WithBaseURL(baseURL string) Option {
return func(cfg *Config) error {
cfg.BaseURL = baseURL
return nil
}
}

@ -1,5 +1,37 @@
package main
import (
"bytes"
text_template "text/template"
)
// RenderPlainText ...
func RenderPlainText(tpl string, ctx interface{}) (string, error) {
t := text_template.Must(text_template.New("tpl").Parse(tpl))
buf := bytes.NewBuffer([]byte{})
err := t.Execute(buf, ctx)
if err != nil {
return "", err
}
return buf.String(), nil
}
const preambleTemplate = `# Twtxt is an open, distributed microblogging platform that
# uses human-readable text files, common transport protocols,
# and free software.
#
# Learn more about twtxt at https://github.com/buckket/twtxt
#
# nick = {{ .Name }}
# url = {{ .URL }}
# source = {{ .Source }}
# avatar = {{ .Avatar }}
# description = {{ .Description }}
# updated_at = {{ .LastModified }}
#
`
const indexTemplate = `
<!DOCTYPE html>
<html lang="en">

@ -10,7 +10,6 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
@ -24,10 +23,6 @@ import (
)
var (
validName = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_\- ]*$`)
ErrInvalidName = errors.New("error: invalid feed name")
ErrNameTooLong = errors.New("error: name is too long")
ErrInvalidImage = errors.New("error: invalid image")
)
@ -134,7 +129,7 @@ func DownloadImage(conf *Config, url string, filename string, opts *ImageOptions
}
}
fn := filepath.Join(conf.Root, filename)
fn := filepath.Join(conf.DataDir, filename)
of, err := os.OpenFile(fn, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {

Loading…
Cancel
Save