Browse Source

Add heuristic for feeds ending with a .txt extension other than twtxt that signifies the feed's cannonical name (e.g: 8ball.txt)

master
James Mills 2 weeks ago
parent
commit
933732f3b9
Signed by: prologic GPG Key ID: AC4C014F1440EBD6
  1. 8
      internal/scraper.go

8
internal/scraper.go

@ -7,6 +7,7 @@ import (
"mime"
"net/http"
"net/url"
"path/filepath"
"regexp"
"strings"
"sync"
@ -298,6 +299,13 @@ func Scrape(conf *Config, req ScrapeRequest) (*ScrapeResult, error) {
} else {
twter.Nick = u.Hostname()
}
} else if strings.HasSuffix(u.Path, ".txt") {
base := filepath.Base(u.Path)
if name := strings.TrimSuffix(base, filepath.Ext(base)); name != "" {
twter.Nick = name
} else {
twter.Nick = u.Hostname()
}
} else {
twter.Nick = Slugify(uri)
}

Loading…
Cancel
Save