Refactor, cleanup debug logging and index type metadata found in feeds to distinguish between user, bot and rss feeds
parent
eb2c6dfbc1
commit
e8444538ad
GPG Key ID:
AC4C014F1440EBD6
@ -1,118 +0,0 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"git.mills.io/yarnsocial/yarns" |
||||
"git.mills.io/yarnsocial/yarns/internal" |
||||
sync "github.com/sasha-s/go-deadlock" |
||||
log "github.com/sirupsen/logrus" |
||||
flag "github.com/spf13/pflag" |
||||
) |
||||
|
||||
var ( |
||||
debug bool |
||||
version bool |
||||
|
||||
path string |
||||
) |
||||
|
||||
const helpText = ` |
||||
geturl retrives a url from the database |
||||
` |
||||
|
||||
func init() { |
||||
baseProg := filepath.Base(os.Args[0]) |
||||
flag.Usage = func() { |
||||
fmt.Fprintf(os.Stderr, "Usage: %s [options] path\n", baseProg) |
||||
fmt.Fprint(os.Stderr, helpText) |
||||
flag.PrintDefaults() |
||||
} |
||||
|
||||
flag.BoolVarP(&debug, "debug", "d", false, "enable debug logging") |
||||
flag.BoolVarP(&version, "version", "v", false, "display version information") |
||||
|
||||
flag.StringVarP(&path, "path", "p", "yarns.db", "path to yarns database") |
||||
} |
||||
|
||||
func flagNameFromEnvironmentName(s string) string { |
||||
s = strings.ToLower(s) |
||||
s = strings.Replace(s, "_", "-", -1) |
||||
return s |
||||
} |
||||
|
||||
func parseArgs() error { |
||||
for _, v := range os.Environ() { |
||||
vals := strings.SplitN(v, "=", 2) |
||||
flagName := flagNameFromEnvironmentName(vals[0]) |
||||
fn := flag.CommandLine.Lookup(flagName) |
||||
if fn == nil || fn.Changed { |
||||
continue |
||||
} |
||||
if err := fn.Value.Set(vals[1]); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
flag.Parse() |
||||
return nil |
||||
} |
||||
|
||||
func main() { |
||||
parseArgs() |
||||
|
||||
if version { |
||||
fmt.Printf("hash_url version %s", yarns.FullVersion()) |
||||
os.Exit(0) |
||||
} |
||||
|
||||
if debug { |
||||
log.SetLevel(log.DebugLevel) |
||||
} else { |
||||
log.SetLevel(log.InfoLevel) |
||||
|
||||
// Disable deadlock detection in production mode
|
||||
sync.Opts.Disable = true |
||||
} |
||||
|
||||
if flag.NArg() < 1 { |
||||
flag.Usage() |
||||
os.Exit(1) |
||||
} |
||||
|
||||
uri := flag.Arg(0) |
||||
|
||||
db, err := internal.NewBitcaskStore(path) |
||||
if err != nil { |
||||
log.WithError(err).Fatal("error opening database") |
||||
} |
||||
if err := db.Merge(); err != nil { |
||||
log.WithError(err).Fatal("error merging store") |
||||
} |
||||
|
||||
url, isNew, err := db.GetOrSetURL(uri) |
||||
if err != nil { |
||||
log.WithError(err).Fatal("error getting url") |
||||
} |
||||
log.Debugf("isNew: %t", isNew) |
||||
|
||||
/* |
||||
hash := internal.HashURL(uri) |
||||
url, err := db.GetURL(hash) |
||||
if err != nil { |
||||
log.WithError(err).Fatal("error getting url") |
||||
} |
||||
*/ |
||||
|
||||
log.Debugf("found url %s with hash %s at %s", url.URL, url.Hash(), url.Key()) |
||||
|
||||
data, err := json.Marshal(url) |
||||
if err != nil { |
||||
log.WithError(err).Fatal("error marshaling json") |
||||
} |
||||
|
||||
fmt.Println(string(data)) |
||||
} |
||||
@ -1,85 +0,0 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"git.mills.io/yarnsocial/yarns" |
||||
"git.mills.io/yarnsocial/yarns/internal" |
||||
sync "github.com/sasha-s/go-deadlock" |
||||
log "github.com/sirupsen/logrus" |
||||
flag "github.com/spf13/pflag" |
||||
) |
||||
|
||||
var ( |
||||
debug bool |
||||
version bool |
||||
) |
||||
|
||||
const helpText = ` |
||||
hashurl hashes a url which is internally used to track feeds already visited |
||||
` |
||||
|
||||
func init() { |
||||
baseProg := filepath.Base(os.Args[0]) |
||||
flag.Usage = func() { |
||||
fmt.Fprintf(os.Stderr, "Usage: %s [options] path\n", baseProg) |
||||
fmt.Fprint(os.Stderr, helpText) |
||||
flag.PrintDefaults() |
||||
} |
||||
|
||||
flag.BoolVarP(&debug, "debug", "d", false, "enable debug logging") |
||||
flag.BoolVarP(&version, "version", "v", false, "display version information") |
||||
} |
||||
|
||||
func flagNameFromEnvironmentName(s string) string { |
||||
s = strings.ToLower(s) |
||||
s = strings.Replace(s, "_", "-", -1) |
||||
return s |
||||
} |
||||
|
||||
func parseArgs() error { |
||||
for _, v := range os.Environ() { |
||||
vals := strings.SplitN(v, "=", 2) |
||||
flagName := flagNameFromEnvironmentName(vals[0]) |
||||
fn := flag.CommandLine.Lookup(flagName) |
||||
if fn == nil || fn.Changed { |
||||
continue |
||||
} |
||||
if err := fn.Value.Set(vals[1]); err != nil { |
||||
return err |
||||
} |
||||
} |
||||
flag.Parse() |
||||
return nil |
||||
} |
||||
|
||||
func main() { |
||||
parseArgs() |
||||
|
||||
if version { |
||||
fmt.Printf("hash_url version %s", yarns.FullVersion()) |
||||
os.Exit(0) |
||||
} |
||||
|
||||
if debug { |
||||
log.SetLevel(log.DebugLevel) |
||||
} else { |
||||
log.SetLevel(log.InfoLevel) |
||||
|
||||
// Disable deadlock detection in production mode
|
||||
sync.Opts.Disable = true |
||||
} |
||||
|
||||
if flag.NArg() < 1 { |
||||
flag.Usage() |
||||
os.Exit(1) |
||||
} |
||||
|
||||
url := flag.Arg(0) |
||||
|
||||
hash := internal.HashURL(url) |
||||
fmt.Println(hash) |
||||
} |
||||
Loading…
Reference in new issue