~q3cpma/mangadex-tools

ref: 0472af001fdf47ef902c0a9a83b7acfce9f79f68 mangadex-tools/mdex_util.tcl -rw-r--r-- 6.8 KiB
0472af00q3cpma Adapt to MDex API change (why don't these monkeys maintain a changelog page for their important API changes?) 2 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
package require json
set scriptdir [file dirname [file dirname \
							 [file normalize [file join [info script] dummy]]]]
source [file join $scriptdir util.tcl]

set URL_BASE       https://mangadex.org
set URL_BASE_RE    https://mangadex\.org
set COVER_SERVER   https://uploads.mangadex.org
set API_URL_BASE   https://api.mangadex.org
set USER_AGENT     {Mozilla/5.0 (Windows NT 6.1; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0}
set UUID_RE        {[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}


# Wrapper to set common curl options
proc curl {args} {
	global USER_AGENT

	exec -ignorestderr curl \
		--compressed \
		--connect-timeout 5 \
		--fail \
		--fail-early \
		--location \
		--max-time 30 \
		--retry 5 \
		--user-agent $USER_AGENT \
		{*}$args
}

# Wrapper around curl to download each URL to the corresponding outname
# args can be used as additional curl options
proc curl_map {urls outnames args} {
	foreach url $urls outname $outnames {
		lappend args -o $outname $url
	}
	curl {*}$args
}

# MangaDex GET API endpoint, with optional query parameters as a dict
proc api_get {endpoint {query_params ""}} {
	global API_URL_BASE

	set args {}
	foreach {key val} $query_params {
		lappend args --data-urlencode $key=$val
	}
	curl --get --no-progress-meter $API_URL_BASE/$endpoint {*}$args
}

# MangaDex JSON POST API endpoint
proc api_post_json {endpoint json} {
	global API_URL_BASE

	curl --request POST --header {Content-Type: application/json} --data $json --no-progress-meter \
		$API_URL_BASE/$endpoint
}

# Convert a Mangadex manga URL to its ID; mode can be "legacy"
proc manga_url_to_id {url {mode ""}} {
	global URL_BASE_RE UUID_RE

	if {$mode eq "legacy"} {
		if {![regexp "^$URL_BASE_RE/title/(\\d+)/\[^/\]+\$" $url -> id]} {
			util::die "$url: invalid legacy URL"
		}
	} else {
		if {![regexp "^$URL_BASE_RE/title/($UUID_RE)\$" $url -> id]} {
			util::die "$url: invalid URL"
		}
	}
	return $id
}

# Helper to get manga title from a relationships list
proc get_rel_title {relationships lang} {
	foreach rel $relationships {
		if {[dict get $rel type] ne "manga"} {
			continue
		}
		if {$lang ne "" && [dict exists $rel attributes title $lang]} {
			return [dict get $rel attributes title $lang]
		} else {
			return [dict get $rel id]
		}
	}
}

# Helper to get scanlation group names from a relationships list
# return {{No Group}} if no group was found
proc get_rel_groups {relationships} {
	set groups [lmap rel $relationships {
		if {[dict get $rel type] ne "scanlation_group"} {
			continue
		}
		dict get $rel attributes name
	}]
	util::? {$groups eq ""} {{No Group}} {$groups}
}

# Get chapter timestamp (using the publishAt field) in `clock seconds` format
proc get_chapter_tstamp {chapter_data} {
	clock scan [regsub {\+\d{2}:\d{2}$} [dict get $chapter_data attributes publishAt] {}] \
		-timezone :UTC -format %Y-%m-%dT%H:%M:%S
}

# Produce a pretty chapter dirname, if title is specified, it overrides the remote one
proc chapter_dirname {chapter_data lang {title ""}} {
	if {$title eq ""} {
		set title [get_rel_title [dict get $chapter_data relationships] $lang]
	}
	set ret "$title - c"
	set num [dict get $chapter_data attributes chapter]
	if {[string is entier -strict $num]} {
		append ret [format %03d $num]
	} elseif {[string is double -strict $num]} {
		append ret [format %05.1f $num]
	} else {
		append ret $num
	}
	set vol [dict get $chapter_data attributes volume]
	if {$vol ne "null"} {
		if {[string is entier -strict $vol]} {
			append ret " (v[format %02d $vol])"
		} elseif {[string is double -strict $vol]} {
			append ret " (v[format %04.1f $vol])"
		} else {
			append ret " (v$vol)"
		}
	}
	set group_names [get_rel_groups [dict get $chapter_data relationships]]
	append ret " \[[join $group_names {, }]\]"
}

# Produce a pretty cover filename, if title is specified, it overrides the remote one
proc cover_filename {cover_data lang {title ""}} {
	if {$title eq ""} {
		set title [get_rel_title [dict get $cover_data relationships] $lang]
	}
	set ret "$title - c000"
	set vol [dict get $cover_data data attributes volume]
	if {$vol ne "null"} {
		if {[string is entier -strict $vol]} {
			append ret " (v[format %02d $vol])"
		} elseif {[string is double -strict $vol]} {
			append ret " (v[format %04.1f $vol])"
		} else {
			append ret " (v$vol)"
		}
	}
	set ext [file extension [dict get $cover_data data attributes fileName]]
	append ret " - Cover$ext"
}

# Get a single chapter from its id
proc get_chapter {cid} {
	set query_params {
		includes[] scanlation_group
		includes[] manga
	}
	puts stderr "Downloading chapter JSON..."
	set chapter [json::json2dict [api_get chapter/$cid $query_params]]
	if {[dict get $chapter result] eq "error"} {
		error [dict get $chapter errors]
	}
	return $chapter
}

# Get the complete chapter list (from smallest to greatest chapter number) for a manga id
# with an optional language filter
proc get_chapter_list {mid lang} {
	set query_params {
		limit          500
		offset         0
		order[chapter] asc
		includes[]     scanlation_group
		includes[]     manga
	}
	if {$lang ne ""} {
		lappend query_params {translatedLanguage[]} $lang
	}
	puts stderr "Downloading manga feed JSON..."
	util::do {
		set manga_feed [json::json2dict [api_get manga/$mid/feed $query_params]]
		dict incr query_params offset 500
		# Filter invalid chapters
		lappend chapters {*}[dict get $manga_feed data]
	} while {[dict get $manga_feed total] - [dict get $query_params offset] > 0}
	return $chapters
}


# Download the pages of a chapters in dirname from its JSON dict
proc dl_chapter {chapter_data dirname} {
	puts stderr "Downloading @Home server URL JSON..."
	set json [api_get at-home/server/[dict get $chapter_data id]]
	set server [dict get [json::json2dict $json] baseUrl]

	set hash [dict get $chapter_data attributes hash]
	set pages [dict get $chapter_data attributes data]

	set urls [util::lprefix $pages $server/data/$hash/]
	set outnames [lmap num [util::iota [llength $pages] 1] page $pages {
		format %0*d%s [string length [llength $pages]] $num [file extension $page]
	}]
	curl_map $urls [util::lprefix $outnames $dirname/]
}

# Download the covers of a manga in cwd. If volumes is specified, download
# only the covers for these
proc dl_covers {mid lang {volumes ""}} {
	global COVER_SERVER

	set query_params {
		limit         100
		offset        0
		order[volume] asc
		includes[]    manga
	}
	lappend query_params {manga[]} $mid

	puts stderr "Downloading cover list JSON..."
	if {[catch {api_get cover $query_params} json]} {
		util::die "Failed to download cover list JSON!\n\n$json"
	}
	set covers [dict get [json::json2dict $json] results]
	foreach cov $covers {
		if {[dict get $cov result] eq "ok" &&
			($volumes eq "" || [dict get $cov data attributes volume] in $volumes)
		} {
			lappend urls $COVER_SERVER/covers/$mid/[dict get $cov data attributes fileName]
			lappend outnames [cover_filename $cov $lang]
		}
	}
	curl_map $urls $outnames
}