Update maxScrapers to 2 (from 50) to slow down scraping and reduce CPU and Disk I/O

1 year ago · 19127bf374
parent 933732f3b9
commit 19127bf374
1 changed files with 1 additions and 1 deletions
--- a/internal/crawl_task.go
+++ b/internal/crawl_task.go
@ -13,7 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 )

-const maxScrapers = 50
+const maxScrapers = 2

 type CrawlTask struct {
 	*BaseTask