From 19127bf374750329d7af02cd8d2afdb554239480 Mon Sep 17 00:00:00 2001 From: James Mills Date: Thu, 28 Oct 2021 08:19:32 +1000 Subject: [PATCH] Update maxScrapers to 2 (from 50) to slow down scraping and reduce CPU and Disk I/O --- internal/crawl_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/crawl_task.go b/internal/crawl_task.go index 48442c4..f645e57 100644 --- a/internal/crawl_task.go +++ b/internal/crawl_task.go @@ -13,7 +13,7 @@ import ( log "github.com/sirupsen/logrus" ) -const maxScrapers = 50 +const maxScrapers = 2 type CrawlTask struct { *BaseTask