~nova/mag-fear-scraper

718b8ed099b3ed4906eb74482079287c7c21d0b8 — Nova Devereaux 1 year, 5 months ago
Current counts
4 files changed, 408 insertions(+), 0 deletions(-)

A build.sh
A episode_fear_list_denormalized.tsv
A episode_fear_list_normalized.tsv
A fear_counts.tsv
A  => build.sh +6 -0
@@ 1,6 @@
#!/bin/sh
wget https://the-magnus-archives.fandom.com/wiki/Episodes
pup 'a[href*=":_"][href*="MAG"] attr{href}' < Episodes | sort -u | xargs -I {} wget https://the-magnus-archives.fandom.com{}
fgrep Related\ Entity MAG* -A 3 | fgrep href | perl -pe 'y/_/ /; s/-//; s/<div.*?">//; s/<\/div>//; s/<a.*?>(.*?)<\/a>[^<]*/$1, /g; s/<.*?>//g; s/(\t.*)(?:Primarily|Possibly) /$1/; s/, $//; $_ .= "\n"' | sort -k 2 -n > episode_fear_list_denormalized.tsv
perl -ne 'chomp; @f = split "\t"; ($id) = ($f[0] =~ /([0-9]*):/); $f[1] =~ s/The //g; @g = split ", ", $f[1]; print "$id\t$_\n" foreach @g' < episode_fear_list_denormalized.tsv > episode_fear_list_normalized.tsv
cut -f 2 episode_fear_list_normalized.tsv | sort | uniq -c | sort -n | sed 's/^ *//' > fear_counts.tsv

A  => episode_fear_list_denormalized.tsv +161 -0
@@ 1,161 @@

MAG 1: Angler Fish	The Stranger
MAG 2: Do Not Open	The Stranger, The Buried
MAG 3: Across the Street	The Stranger, The Web
MAG 4: Page Turner	The Vast, The Spiral, The End
MAG 5: Thrown Away	The Flesh
MAG 6: Squirm	The Corruption
MAG 7: The Piper	The Slaughter
MAG 8: Burned Out	The Web, The Desolation, The Spiral
MAG 9: A Fathers Love	The Dark, The Hunt
MAG 10: Vampire Killer	The Hunt
MAG 11: Dreamer	The End
MAG 12: First Aid	The Desolation, The Eye
MAG 13: Alone	The Lonely
MAG 14: Piecemeal	The Flesh
MAG 15: Lost Johns' Cave	The Buried
MAG 16: Arachnophobia	The Web, The Corruption
MAG 17: The Boneturner's Tale	The Flesh
MAG 18: The Man Upstairs	The Flesh
MAG 19: Confession	The Spiral, The Web, The Desolation
MAG 20: Desecrated Host	The Spiral, The Flesh
MAG 21: Freefall	The Vast
MAG 22: Colony	The Corruption
MAG 23: Schwartzwald	The Eye
MAG 24: Strange Music	The Stranger
MAG 25: Growing Dark	The Dark
MAG 26: A Distortion	The Spiral, The Corruption
MAG 27: A Sturdy Lock	The Spiral
MAG 28: Skintight	The Slaughter, The Stranger
MAG 29: Cheating Death	The End
MAG 30: Killing Floor	The Flesh
MAG 31: First Hunt	The Hunt
MAG 32: Hive	The Corruption
MAG 33: Boatswain's Call	The Lonely
MAG 34: Anatomy Class	The Stranger
MAG 36: Taken Ill	The Corruption, The Hunt
MAG 37: Burnt Offering	The Desolation
MAG 38: Lost and Found	The Spiral
MAG 39: Infestation	The Corruption, The Stranger
MAG 40: Human Remains	The Eye, The Corruption, The Stranger
MAG 41: Too Deep	The Buried, The Dark
MAG 42: Grifter's Bone	The Slaughter
MAG 43: Section 31	The Desolation, The End
MAG 44: Tightrope	The Stranger
MAG 45: Blood Bag	The Corruption
MAG 46: Literary Heights	The Spiral, The Vast
MAG 47: The New Door	The Spiral
MAG 48: Lost in the Crowd	The Lonely
MAG 49: The Butchers Window	The Flesh
MAG 50: Foundations	The Buried
MAG 51: High Pressure	The Vast, The Buried
MAG 52: Exceptional Risk	The Dark
MAG 53: Crusader	The Eye
MAG 54: Still Life	The Stranger
MAG 55: Pest Control	The Corruption, The Desolation
MAG 56: Children of the Night	The Hunt, The Web
MAG 57: Personal Space	The Lonely, The Vast
MAG 58: Trail Rations	The Flesh
MAG 59: Recluse	The Web, The Desolation
MAG 60: Observer Effect	The Eye
MAG 61: Hard Shoulder	The Hunt, The Stranger, The Buried
MAG 62: First Edition	The End, The Eye
MAG 63: The End of the Tunnel	The Dark
MAG 64: Burial Rites	The End
MAG 65: Binary	The Spiral, The Extinction
MAG 66: Held in Customs	The Buried
MAG 67: Burning Desire	The Desolation
MAG 68: The Tale of a Field Hospital	The Corruption
MAG 69: Thought for the Day	The Web
MAG 70: Book of the Dead	The End
MAG 71: Underground	The Buried
MAG 72: Takeaway	The Flesh
MAG 73: Police Lights	The Dark
MAG 74: Fatigue	The Spiral
MAG 75: A Long Way Down	The Vast
MAG 76: The Smell of Blood	The Slaughter
MAG 77: The Kind Mother	The Stranger
MAG 78: Distant Cousin	The Stranger, The Web, The Spiral
MAG 79: Hide and Seek	The Stranger, The Spiral
MAG 81: A Guest For Mr. Spider	The Web
MAG 82: The Eyewitnesses	The Eye, The Slaughter
MAG 83: Drawing A Blank	The Stranger
MAG 84: Possessive	The Corruption
MAG 85: Upon the Stair	The Spiral
MAG 86: Tucked In	The Dark
MAG 87: The Uncanny Valley	The Stranger, The Desolation
MAG 88: Dig	The Buried
MAG 89: Twice as Bright	The Desolation
MAG 90: Body Builder	The Flesh
MAG 91: The Coming Storm	The Vast, The Spiral
MAG 92: Nothing Beside Remains	The Eye, The Lonely
MAG 93: Contaminant	The Corruption
MAG 94: Dead Woman Walking	The End
MAG 95: Absent Without Leave	The Slaughter, The End, The Corruption
MAG 96: Return to Sender	The Stranger
MAG 97: We All Ignore the Pit	The Buried, The Vast
MAG 98: Lights Out	The Dark
MAG 99: Dust to Dust	The Buried
MAG 100: I Guess You Had To Be There	The Desolation, The Dark, The Spiral, The Web, The Lonely
MAG 101: Another Twist	The Stranger, The Spiral
MAG 102: Nesting Instinct	The Corruption
MAG 103: Cruelty Free	The Flesh
MAG 104: Sneak Preview	The Stranger
MAG 105: Total War	The Slaughter, The Eye
MAG 106: A Matter Of Perspective	The Vast, The Eye
MAG 107: Third Degree	The Desolation, The Hunt
MAG 108: Monologue	The Lonely, The Stranger
MAG 109: Nightfall	The Dark, The Hunt
MAG 110: Creature Feature	The Web
MAG 111: Family Business	The End
MAG 112: Thrill of the Chase	The Hunt
MAG 113: Breathing Room	The End
MAG 114: Cracked Foundation	The Web, The Stranger, The Extinction
MAG 115: Taking Stock	The Flesh
MAG 116: The Show Must Go On	The Stranger, The Slaughter
MAG 117: Testament	The Eye
MAG 118: The Masquerade	The Stranger, The Eye
MAG 119: Stranger and Stranger	The Stranger
MAG 120: Eye Contact	The Eye
MAG 121: Far Away	The End, The Web
MAG 122: Zombie	The Stranger
MAG 123: Web Development	The Web
MAG 124: Left Hanging	The Vast
MAG 125: Civilian Casualties	The Slaughter
MAG 126: Sculptor's Tool	The Spiral
MAG 127: Remains to be Seen	The Eye
MAG 128: Heavy Goods	The Stranger
MAG 129: Submerged	The Buried
MAG 130: Meat	The Flesh
MAG 131: Flesh	The Flesh, The Spiral
MAG 132: Entombed	The Buried, The Eye
MAG 133: Dead Horse	The Hunt
MAG 134: Time of Revelation	The Extinction
MAG 135: Dark Matter	The Dark
MAG 136: The Puppeteer	The Web
MAG 137: Nemesis	The Slaughter
MAG 138: The Architecture of Fear	The Eye
MAG 139: Chosen	The Desolation
MAG 140: The Movement of the Heavens	The Dark
MAG 141: Doomed Voyage	The Vast, The Spiral
MAG 142: Scrutiny	The Eye, The Buried
MAG 143: Heart of Darkness	The Dark
MAG 144: Decrypted	The Extinction, The Lonely
MAG 145: Infectious Doubts	The Desolation
MAG 146: Threshold	The Spiral
MAG 147: Weaver	The Web
MAG 148: Extended Surveillance	The Eye
MAG 149: Concrete Jungle	The Extinction
MAG 150: CulDe-Sac-	The Lonely
MAG 151: Big Picture	The Vast, The Lonely, The Extinction
MAG 152: A Gravedigger's Envy	The Buried
MAG 153: Love Bombing	The Corruption, Flesh
MAG 154: Bloody Mary	The Eye
MAG 155: Cost of Living	The End
MAG 156: Reflection	The Extinction
MAG 157: Rotten Core	The Extinction, The Corruption
MAG 158: Panopticon	The Eye, The Extinction, The Lonely
MAG 159: The Last	The Lonely
MAG 160: The Eye Opens	The Eye
MAG 163: In the Trenches	The Slaughter
MAG 164: The Sick Village	The Corruption

A  => episode_fear_list_normalized.tsv +226 -0
@@ 1,226 @@
1	Stranger
2	Stranger
2	Buried
3	Stranger
3	Web
4	Vast
4	Spiral
4	End
5	Flesh
6	Corruption
7	Slaughter
8	Web
8	Desolation
8	Spiral
9	Dark
9	Hunt
10	Hunt
11	End
12	Desolation
12	Eye
13	Lonely
14	Flesh
15	Buried
16	Web
16	Corruption
17	Flesh
18	Flesh
19	Spiral
19	Web
19	Desolation
20	Spiral
20	Flesh
21	Vast
22	Corruption
23	Eye
24	Stranger
25	Dark
26	Spiral
26	Corruption
27	Spiral
28	Slaughter
28	Stranger
29	End
30	Flesh
31	Hunt
32	Corruption
33	Lonely
34	Stranger
36	Corruption
36	Hunt
37	Desolation
38	Spiral
39	Corruption
39	Stranger
40	Eye
40	Corruption
40	Stranger
41	Buried
41	Dark
42	Slaughter
43	Desolation
43	End
44	Stranger
45	Corruption
46	Spiral
46	Vast
47	Spiral
48	Lonely
49	Flesh
50	Buried
51	Vast
51	Buried
52	Dark
53	Eye
54	Stranger
55	Corruption
55	Desolation
56	Hunt
56	Web
57	Lonely
57	Vast
58	Flesh
59	Web
59	Desolation
60	Eye
61	Hunt
61	Stranger
61	Buried
62	End
62	Eye
63	Dark
64	End
65	Spiral
65	Extinction
66	Buried
67	Desolation
68	Corruption
69	Web
70	End
71	Buried
72	Flesh
73	Dark
74	Spiral
75	Vast
76	Slaughter
77	Stranger
78	Stranger
78	Web
78	Spiral
79	Stranger
79	Spiral
81	Web
82	Eye
82	Slaughter
83	Stranger
84	Corruption
85	Spiral
86	Dark
87	Stranger
87	Desolation
88	Buried
89	Desolation
90	Flesh
91	Vast
91	Spiral
92	Eye
92	Lonely
93	Corruption
94	End
95	Slaughter
95	End
95	Corruption
96	Stranger
97	Buried
97	Vast
98	Dark
99	Buried
100	Desolation
100	Dark
100	Spiral
100	Web
100	Lonely
101	Stranger
101	Spiral
102	Corruption
103	Flesh
104	Stranger
105	Slaughter
105	Eye
106	Vast
106	Eye
107	Desolation
107	Hunt
108	Lonely
108	Stranger
109	Dark
109	Hunt
110	Web
111	End
112	Hunt
113	End
114	Web
114	Stranger
114	Extinction
115	Flesh
116	Stranger
116	Slaughter
117	Eye
118	Stranger
118	Eye
119	Stranger
120	Eye
121	End
121	Web
122	Stranger
123	Web
124	Vast
125	Slaughter
126	Spiral
127	Eye
128	Stranger
129	Buried
130	Flesh
131	Flesh
131	Spiral
132	Buried
132	Eye
133	Hunt
134	Extinction
135	Dark
136	Web
137	Slaughter
138	Eye
139	Desolation
140	Dark
141	Vast
141	Spiral
142	Eye
142	Buried
143	Dark
144	Extinction
144	Lonely
145	Desolation
146	Spiral
147	Web
148	Eye
149	Extinction
150	Lonely
151	Vast
151	Lonely
151	Extinction
152	Buried
153	Corruption
153	Flesh
154	Eye
155	End
156	Extinction
157	Extinction
157	Corruption
158	Eye
158	Extinction
158	Lonely
159	Lonely
160	Eye
163	Slaughter
164	Corruption

A  => fear_counts.tsv +15 -0
@@ 1,15 @@
9 Extinction
10 Hunt
11 Slaughter
12 Lonely
12 Vast
13 Dark
13 End
14 Desolation
15 Buried
15 Flesh
16 Web
18 Corruption
21 Eye
21 Spiral
26 Stranger