Browse Source

Add support for aggregating Twitter timelines into Twtxt feeds (#12)

See https://twtxt.net/conv/gtr5w5q

Co-authored-by: James Mills <prologic@shortcircuit.net.au>
Reviewed-on: #12
Co-authored-by: James Mills <james@mills.io>
Co-committed-by: James Mills <james@mills.io>
master
James Mills 2 months ago
parent
commit
708acd671e
  1. 1
      config.go
  2. 156
      feeds.go
  3. 4
      go.mod
  4. 23
      go.sum
  5. 37
      handlers.go
  6. 20
      jobs.go
  7. 22
      main.go
  8. 35
      templates.go
  9. 35
      uri.go
  10. 9
      utils.go

1
config.go

@ -24,6 +24,7 @@ func (conf *Config) Save() error {
if err != nil {
return err
}
data = append([]byte("---\n"), data...)
return ioutil.WriteFile(conf.path, data, 0644)
}

156
feeds.go

@ -1,23 +1,29 @@
package main
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/andyleap/microformats"
"github.com/gosimple/slug"
"github.com/mmcdole/gofeed"
twitterscraper "github.com/n0madic/twitter-scraper"
log "github.com/sirupsen/logrus"
)
const (
avatarResolution = 60 // 60x60 px
rssTwtxtTemplate = "%s\t%s ⌘ [Read more...](%s)\n"
twtxtTemplate = "%s\t%s ⌘ [Read more](%s)\n"
maxTwtLength = 288
maxTweets = 10
)
var (
@ -27,14 +33,44 @@ var (
// Feed ...
type Feed struct {
Name string
URL string
URI string
LastModified string
}
func TestFeed(url string) (*gofeed.Feed, error) {
func ProcessFeedContent(title, desc string, max int) string {
converter := md.NewConverter("", true, nil)
markdown, err := converter.ConvertString(desc)
if err != nil {
log.WithError(err).Warnf("error converting content to html")
return fmt.Sprintf("%s: %s", title, err)
}
return CleanTwt(fmt.Sprintf("**%s**\n%s", title, markdown))[:max]
}
func TestTwitterFeed(handle string) error {
count := 0
for tweet := range twitterscraper.WithReplies(false).GetTweets(context.Background(), handle, maxTweets) {
if tweet.Error != nil {
return fmt.Errorf("error scraping tweets from %s: %w", handle, tweet.Error)
}
if tweet.IsRetweet {
continue
}
count++
}
if count == 0 {
log.WithField("handle", handle).WithField("handle", handle).Warn("empty or bad twitter handle")
}
return nil
}
func TestRSSFeed(uri string) (*gofeed.Feed, error) {
fp := gofeed.NewParser()
feed, err := fp.ParseURL(url)
feed, err := fp.ParseURL(uri)
if err != nil {
return nil, err
}
@ -42,7 +78,7 @@ func TestFeed(url string) (*gofeed.Feed, error) {
return feed, nil
}
func FindFeed(uri string) (*gofeed.Feed, string, error) {
func FindRSSFeed(uri string) (*gofeed.Feed, string, error) {
u, err := url.Parse(uri)
if err != nil {
return nil, "", err
@ -62,41 +98,54 @@ func FindFeed(uri string) (*gofeed.Feed, string, error) {
altMap[alt.Type] = alt.URL
}
feedURL := altMap["application/atom+xml"]
feedURI := altMap["application/atom+xml"]
if feedURL == "" {
if feedURI == "" {
for _, alt := range data.Alternates {
switch alt.Type {
case "application/atom+xml", "application/rss+xml":
feedURL = alt.URL
feedURI = alt.URL
break
}
}
}
if feedURL == "" {
if feedURI == "" {
return nil, "", ErrNoSuitableFeedsFound
}
feed, err := TestFeed(feedURL)
feed, err := TestRSSFeed(feedURI)
if err != nil {
return nil, "", err
}
return feed, feedURL, nil
return feed, feedURI, nil
}
// ValidateTwitterFeed ...
func ValidateTwitterFeed(conf *Config, handle string) (Feed, error) {
err := TestTwitterFeed(handle)
if err != nil {
log.WithError(err).Warnf("invalid twitter feed %s", handle)
}
name := fmt.Sprintf("twitter-%s", handle)
uri := fmt.Sprintf("twitter://%s", handle)
return Feed{Name: name, URI: uri}, nil
}
// ValidateFeed ...
func ValidateFeed(conf *Config, url string) (Feed, error) {
feed, err := TestFeed(url)
func ValidateRSSFeed(conf *Config, uri string) (Feed, error) {
feed, err := TestRSSFeed(uri)
if err != nil {
log.WithError(err).Warnf("invalid feed %s", url)
log.WithError(err).Warnf("invalid rss feed %s", uri)
}
if feed == nil {
feed, url, err = FindFeed(url)
feed, uri, err = FindRSSFeed(uri)
if err != nil {
log.WithError(err).Errorf("no feed found on %s", url)
log.WithError(err).Errorf("no rss feeds found on %s", uri)
return Feed{}, err
}
}
@ -117,10 +166,77 @@ func ValidateFeed(conf *Config, url string) (Feed, error) {
}
}
return Feed{Name: name, URL: url}, nil
return Feed{Name: name, URI: uri}, nil
}
// Code borrowed from https://github.com/n0madic/twitter2rss
// With permission from the author: https://github.com/n0madic/twitter2rss/issues/3
func UpdateTwitterFeed(conf *Config, name, handle string) error {
var lastModified = time.Time{}
fn := filepath.Join(conf.Root, fmt.Sprintf("%s.txt", name))
stat, err := os.Stat(fn)
if err == nil {
lastModified = stat.ModTime()
}
f, err := os.OpenFile(fn, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
return err
}
defer f.Close()
old, new := 0, 0
for tweet := range twitterscraper.WithReplies(false).GetTweets(context.Background(), handle, maxTweets) {
if tweet.Error != nil {
return fmt.Errorf("error scraping tweets from %s: %w", handle, tweet.Error)
}
if tweet.IsRetweet {
continue
}
if tweet.TimeParsed.After(lastModified) {
var title string
titleSplit := strings.FieldsFunc(tweet.Text, func(r rune) bool {
return r == '\n' || r == '!' || r == '?' || r == ':' || r == '<' || r == '.' || r == ','
})
if len(titleSplit) > 0 {
if strings.HasPrefix(titleSplit[0], "a href") || strings.HasPrefix(titleSplit[0], "http") {
title = "link"
} else {
title = titleSplit[0]
}
}
title = strings.TrimSuffix(title, "https")
title = strings.TrimSpace(title)
text := fmt.Sprintf(
twtxtTemplate,
tweet.TimeParsed.Format(time.RFC3339),
ProcessFeedContent(title, tweet.HTML, maxTwtLength-len(tweet.PermanentURL)),
tweet.PermanentURL,
)
_, err := f.WriteString(text)
if err != nil {
return err
}
} else {
old++
}
}
if (old + new) == 0 {
log.WithField("name", name).WithField("handle", handle).Warn("empty or bad twitter handle")
}
return nil
}
func UpdateFeed(conf *Config, name, url string) error {
func UpdateRSSFeed(conf *Config, name, url string) error {
fp := gofeed.NewParser()
feed, err := fp.ParseURL(url)
if err != nil {
@ -166,9 +282,9 @@ func UpdateFeed(conf *Config, name, url string) error {
if item.PublishedParsed.After(lastModified) {
new++
text := fmt.Sprintf(
rssTwtxtTemplate,
twtxtTemplate,
item.PublishedParsed.Format(time.RFC3339),
item.Title,
ProcessFeedContent(item.Title, item.Description, maxTwtLength-len(item.Link)),
item.Link,
)
_, err := f.WriteString(text)

4
go.mod

@ -3,6 +3,7 @@ module git.mills.io/prologic/rss2twtxt
go 1.14
require (
github.com/JohannesKaufmann/html-to-markdown v1.3.0 // indirect
github.com/andyleap/microformats v0.0.0-20150523144534-25ae286f528b
github.com/aofei/cameron v1.1.5
github.com/divan/num2words v0.0.0-20170904212200-57dba452f942
@ -14,14 +15,13 @@ require (
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/kr/pretty v0.2.0 // indirect
github.com/mmcdole/gofeed v1.0.0
github.com/n0madic/twitter-scraper v0.0.0-20210730114638-48484955d4d4 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/rickb777/accept v0.0.0-20170318132422-d5183c44530d
github.com/robfig/cron v1.2.0
github.com/sirupsen/logrus v1.4.2
github.com/spf13/pflag v1.0.3
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect
golang.org/x/text v0.3.2 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/yaml.v2 v2.2.2 // indirect
)

23
go.sum

@ -1,7 +1,13 @@
github.com/JohannesKaufmann/html-to-markdown v1.3.0 h1:K/p4cq8Ib13hcSVcKQNfKCSWw93CYW5pAjY0fl85has=
github.com/JohannesKaufmann/html-to-markdown v1.3.0/go.mod h1:JNSClIRYICFDiFhw6RBhBeWGnMSSKVZ6sPQA+TK4tyM=
github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/andyleap/microformats v0.0.0-20150523144534-25ae286f528b h1:jnCPxFuWTxrUk9L7/0VIFL0mQGFFSwbH0sfQ7XwsTYg=
github.com/andyleap/microformats v0.0.0-20150523144534-25ae286f528b/go.mod h1:I3yyaN+QdpdChOtQg3ApgY01JRmFsXJASweq6Ye5A3s=
github.com/aofei/cameron v1.1.5 h1:AS7jy+cRk8FahjGV5fPiUCOVVAbqkB1UIZBflqKTzh0=
@ -18,6 +24,7 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gosimple/slug v1.9.0 h1:r5vDcYrFz9BmfIAMC829un9hq7hKM4cHUrsv36LbEqs=
@ -27,6 +34,7 @@ github.com/h2non/filetype v1.1.0/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@ -36,8 +44,11 @@ github.com/mmcdole/gofeed v1.0.0 h1:PHqwr8fsEm8xarj9s53XeEAFYhRM3E9Ib7Ie766/LTE=
github.com/mmcdole/gofeed v1.0.0/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
github.com/n0madic/twitter-scraper v0.0.0-20210730114638-48484955d4d4 h1:FI0825GMw12Dco0XzcvdXX7MBCC8ljIYFYNodIt0idc=
github.com/n0madic/twitter-scraper v0.0.0-20210730114638-48484955d4d4/go.mod h1:uzFP7WK9gCykBnULzfmxScXgIbs+423tWjbHRLYVq/g=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ=
@ -46,6 +57,9 @@ github.com/rickb777/accept v0.0.0-20170318132422-d5183c44530d h1:BhTnJzAi1hrLiyT
github.com/rickb777/accept v0.0.0-20170318132422-d5183c44530d/go.mod h1:sv64uV+hMk2K4qwURvESkYmF8QyMYF/9nJpxF8UPQb8=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/sebdah/goldie/v2 v2.5.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
@ -53,8 +67,11 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.2.0/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -62,6 +79,9 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200320220750-118fecf932d8 h1:1+zQlQqEEhUeStBTi653GZAnAuivZq/2hz+Iz+OP7rg=
golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
@ -71,11 +91,14 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

37
handlers.go

@ -65,19 +65,40 @@ func (app *App) IndexHandler(w http.ResponseWriter, r *http.Request) {
}
if r.Method == http.MethodPost {
url := r.FormValue("url")
uri := r.FormValue("uri")
if url == "" {
if err := renderMessage(w, http.StatusBadRequest, "Error", "No url supplied"); err != nil {
if uri == "" {
if err := renderMessage(w, http.StatusBadRequest, "Error", "No uri supplied"); err != nil {
log.WithError(err).Error("error rendering message template")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
return
}
feed, err := ValidateFeed(app.conf, url)
u, err := ParseURI(uri)
if err != nil {
if err := renderMessage(w, http.StatusBadRequest, "Error", fmt.Sprintf("Unable to find a valid RSS/Atom feed for: %s", url)); err != nil {
log.WithError(err).Errorf("error parsing feed %s", uri)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
var feed Feed
switch u.Type {
case "rss", "http", "https":
feed, err = ValidateRSSFeed(app.conf, uri)
case "twitter":
feed, err = ValidateTwitterFeed(app.conf, u.Config)
default:
if err := renderMessage(w, http.StatusBadRequest, "Error", "Unsupproted feed"); err != nil {
log.WithError(err).Error("error rendering message template")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
return
}
if err != nil {
if err := renderMessage(w, http.StatusBadRequest, "Error", fmt.Sprintf("Unable to find a valid RSS/Atom feed for: %s", uri)); err != nil {
log.WithError(err).Error("error rendering message template")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
@ -92,7 +113,7 @@ func (app *App) IndexHandler(w http.ResponseWriter, r *http.Request) {
return
}
app.conf.Feeds[feed.Name] = feed.URL
app.conf.Feeds[feed.Name] = feed.URI
if err := app.conf.Save(); err != nil {
msg := fmt.Sprintf("Could not save feed: %s", err)
if err := renderMessage(w, http.StatusInternalServerError, "Error", msg); err != nil {
@ -102,7 +123,7 @@ func (app *App) IndexHandler(w http.ResponseWriter, r *http.Request) {
return
}
msg := fmt.Sprintf("Feed successfully added %s: %s", feed.Name, feed.URL)
msg := fmt.Sprintf("Feed successfully added %s: %s", feed.Name, feed.URI)
if err := renderMessage(w, http.StatusCreated, "Success", msg); err != nil {
log.WithError(err).Error("error rendering message template")
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@ -252,7 +273,7 @@ func (app *App) WeAreFeedsHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
for _, feed := range app.GetFeeds() {
fmt.Fprintf(w, "%s %s\n", feed.Name, feed.URL)
fmt.Fprintf(w, "%s %s\n", feed.Name, feed.URI)
}
return
}

20
jobs.go

@ -90,9 +90,23 @@ func NewUpdateFeedsJob(conf *Config) cron.Job {
func (job *UpdateFeedsJob) Run() {
conf := job.conf
for name, url := range conf.Feeds {
if err := UpdateFeed(conf, name, url); err != nil {
log.WithError(err).Errorf("error updating feed %s: %s", name, url)
for name, uri := range conf.Feeds {
u, err := ParseURI(uri)
if err != nil {
log.WithError(err).Errorf("error parsing feed %s: %s", name, uri)
} else {
switch u.Type {
case "rss", "http", "https":
if err := UpdateRSSFeed(conf, name, uri); err != nil {
log.WithError(err).Errorf("error updating rss feed %s: %s", name, uri)
}
case "twitter":
if err := UpdateTwitterFeed(conf, name, u.Config); err != nil {
log.WithError(err).Errorf("error updating twitter feed %s: %s", name, uri)
}
default:
log.Warnf("error unknown feed type %s: %s", name, uri)
}
}
}
}

22
main.go

@ -56,10 +56,26 @@ func main() {
os.Exit(0)
}
url := flag.Arg(0)
uri := flag.Arg(0)
name := flag.Arg(1)
if err := UpdateFeed(&Config{Root: "."}, name, url); err != nil {
log.WithError(err).Fatal("error updating feed")
conf := &Config{Root: "."}
u, err := ParseURI(uri)
if err != nil {
log.WithError(err).Errorf("error parsing feed %s: %s", name, uri)
} else {
switch u.Type {
case "rss", "http", "https":
if err := UpdateRSSFeed(conf, name, uri); err != nil {
log.WithError(err).Errorf("error updating rss feed %s: %s", name, uri)
}
case "twitter":
if err := UpdateTwitterFeed(conf, name, u.Config); err != nil {
log.WithError(err).Errorf("error updating twitter feed %s: %s", name, uri)
}
default:
log.Warnf("error unknown feed type %s: %s", name, uri)
}
}
}

35
templates.go

@ -6,46 +6,41 @@ const indexTemplate = `
<head>
<link rel="stylesheet" href="https://unpkg.com/@picocss/pico@latest/css/pico.min.css">
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>rss2twtxt :: {{ .Title }}</title>
<title>twtxtfeeds :: {{ .Title }}</title>
</head>
<body>
<nav class="container-fluid">
<ul>
<li><strong><a href="/">rss2twtxt</a></strong></li>
<li><strong><a href="/">TwtxtFeeds</a></strong></li>
<li><a href="/feeds">Feeds</a></li>
</ul>
</nav>
<main class="container">
<article class="grid">
<div>
<hgroup>
<h2>rss2twtxt</h2>
<footer>RSS/Atom to twtxt feed</footer>
</hgroup>
<div class="container-fluid">
<form action="/" method="POST">
<input type="uri" id="uri" name="uri" placeholder="Enter any website URL, RSS feed URI or twitter://<handle>" required>
<div><button type="submit">Go!</button>
</form>
</div>
<p>
rss2twtxt is a command-line tool and web app that processes RSS and Atom feeds
twtxtfeeds is a command-line tool and web app that processes RSS, Atom and Twitter feeds
into <a href="https://twtxt.readthedocs.io/en/stable/index.html">twtxt</a>
feeds for consumption by <i>twtxt</i> clients such as <a href="https://twtxt.net">twtxt.net</a>
and <a href="https://twt.social">Twt.social</a> pods.
and <a href="https://yarn.social">Yarn.social</a> pods.
</p>
<p>
You may freely create new feeds here by simply dropping a website's URL
into the form below and a valid RSS/Atom feed will be automagically
discovered and if valid added to the list of <a href="/feeds">/feeds</a>.
You may freely create new feeds here by simply dropping a website's URL,
any valid RSS/Atom URL or a Twitter handle in the form of <code>twitter://<handle></code>.
</p>
<p>
You are also welcome to subscribe to any of the <a href="/feeds">feeds</a>
with your favorite <i>twtxt</i> client (<i>I like using
<a href="https://github.com/quite/twet">twet</a></i>).
Be sure to check out <a href="https://twtxt.net">twtxt.net</a>
and <a href="https://twt.social">Twt.social</a> pods.
and <a href="https://yarn.social">Yarn.social</a> pods.
</p>
<div class="container-fluid">
<form action="/" method="POST">
<input type="url" id="url" name="url" placeholder="Enter any website URL here" required>
<div><button type="submit">Add</button>
</form>
</div>
</div>
</article>
</main>
@ -67,7 +62,7 @@ const feedsTemplate = `
<head>
<link rel="stylesheet" href="https://unpkg.com/@picocss/pico@latest/css/pico.min.css">
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>rss2twtxt :: {{ .Title }}</title>
<title>twtxtfeeds :: {{ .Title }}</title>
</head>
<body>
<nav class="container-fluid">
@ -86,7 +81,7 @@ const feedsTemplate = `
{{ if .Feeds }}
<ul>
{{ range .Feeds }}
<li><a href="{{ .URL }}">{{ .Name }}</a>&nbsp;<small>({{ .LastModified }})</small></li>
<li><a href="{{ .URI }}">{{ .Name }}</a>&nbsp;<small>({{ .LastModified }})</small></li>
{{ end }}
</ul>
{{ else }}

35
uri.go

@ -0,0 +1,35 @@
package main
import (
"fmt"
"strings"
)
type URI struct {
Type string
SubType string
Config string
}
func (u *URI) String() string {
if u.SubType != "" {
return fmt.Sprintf("%s+%s://%s", u.Type, u.SubType, u.Config)
}
return fmt.Sprintf("%s://%s", u.Type, u.Config)
}
func ParseURI(uri string) (*URI, error) {
parts := strings.Split(uri, "://")
if len(parts) == 2 {
types := strings.Split(parts[0], "+")
if len(types) == 2 {
return &URI{
Type: strings.ToLower(types[0]),
SubType: strings.ToLower(types[1]),
Config: parts[1],
}, nil
}
return &URI{Type: parts[0], Config: parts[1]}, nil
}
return nil, fmt.Errorf("invalid uri: %s", uri)
}

9
utils.go

@ -31,6 +31,15 @@ var (
ErrInvalidImage = errors.New("error: invalid image")
)
// CleanTwt cleans a twt's text, replacing new lines with spaces and
// stripping surrounding spaces.
func CleanTwt(text string) string {
text = strings.TrimSpace(text)
text = strings.ReplaceAll(text, "\r\n", "\n")
text = strings.ReplaceAll(text, "\n", "\u2028")
return text
}
func RotateFile(fn string) error {
now := time.Now().Unix()
return os.Rename(fn, fmt.Sprintf("%s.%d", fn, now))

Loading…
Cancel
Save