~q3cpma/mangadex-tools

1b922d4cfeb43f3bd57245e34b98e3227a22d195 — q3cpma 8 months ago 2ad3325
Fix lack of rate limiting in mdex_dl and use/make a generic rate limiting mechanism
3 files changed, 16 insertions(+), 4 deletions(-)

M mdex_dl.tcl
M mdex_monitor.tcl
M mdex_util.tcl
M mdex_dl.tcl => mdex_dl.tcl +0 -3
@@ 62,9 62,6 @@ if {[regexp "^$URL_BASE_RE/title/($UUID_RE)(?:/\[^/\]+)?\$" [lindex $argv 0] -> 
			puts stderr "Failed to download chapter $cid JSON\n\n$chapter"
			continue
		}
		if {$argc > 5} {
			after 300; # Sleep to avoid hitting the rate limit of 5 req/s
		}
		set chapter
	}]
}

M mdex_monitor.tcl => mdex_monitor.tcl +0 -1
@@ 173,7 173,6 @@ foreach entry $catalog {
		puts stderr [util::? {$no_local_tstamp} \
						 "New catalog item, monitoring chapter updates from now on" \
						 "No new chapters"]
		after 300; # Sleep to avoid hitting the rate limit of 5 req/s
		continue
	}
	# Filter chapters to keep only the new ones

M mdex_util.tcl => mdex_util.tcl +16 -0
@@ 36,10 36,26 @@ proc curl_map {urls outnames args} {
	curl {*}$args
}

set RL_state [dict create]
# Enforce a rate limit of reqnum requests per durations ms
proc rate_limit {ident reqnum duration} {
	global RL_state

	set now [clock milliseconds]
	if {[dict get? $RL_state fifo $ident] && [llength $fifo] == $reqnum} {
		dict set RL_state $ident [lassign $fifo last]
		if {[- $now $last] < $duration} {
			after [- $duration [- $now $last]]
		}
	}
	dict lappend RL_state $ident $now
}

# MangaDex GET API endpoint, with optional query parameters as a dict
proc api_get {endpoint {query_params ""}} {
	global API_URL_BASE

	rate_limit global 5 1200; # Actual rate limit is 5 req/s
	set args {}
	foreach {key val} $query_params {
		lappend args --data-urlencode $key=$val