~elektito/gemplex

09d6aecb0fe8ebd8956d28c8ee0c7b6dcc503b7f — Mostafa Razavi 7 months ago 4ce65ef
Fix robots.txt back off

We were first checking robots_prefixes being null, before checking retry
time. So in case robots_prefixes _was_ actually null, we wouldn't back
off.
1 files changed, 5 insertions(+), 5 deletions(-)

M cmd/gemplex/crawl.go
M cmd/gemplex/crawl.go => cmd/gemplex/crawl.go +5 -5
@@ 19,10 19,10 @@ import (
	"sync"
	"time"

	"github.com/a-h/gemini"
	"git.sr.ht/~elektito/gemplex/pkg/gcrawler"
	"git.sr.ht/~elektito/gemplex/pkg/gparse"
	"git.sr.ht/~elektito/gemplex/pkg/utils"
	"github.com/a-h/gemini"
)

const (


@@ 632,13 632,13 @@ where hostname = $1`
	}
	utils.PanicOnErr(err)

	if !prefixesStr.Valid {
		err = fmt.Errorf("No prefixes available")
	if nextTryTime.Time.After(time.Now()) {
		err = ErrRobotsBackoff
		return
	}

	if nextTryTime.Time.After(time.Now()) {
		err = ErrRobotsBackoff
	if !prefixesStr.Valid {
		err = fmt.Errorf("No prefixes available")
		return
	}