~nhanb/pytaku: remove cloudscraper

4 files changed, 23 insertions(+), 51 deletions(-)

M README.md
M poetry.lock
M pyproject.toml
M src/mangoapi/base_site.py

M README.md => README.md +16 -10

@@ 28,8 28,6 @@ On Chapter page, press `?` to show keyboard shortcuts.
 
 ```sh
 ## Backend ##
-doas pacman -S nodejs  # used by cloudscraper to bypass Cloudflare
-
 poetry install
 pip install --upgrade pip
 pip install https://github.com/rogerbinns/apsw/releases/download/3.34.0-r1/apsw-3.34.0-r1.zip \


@@ 49,7 47,7 @@ pytaku-scheduler  # scheduled tasks e.g. update titles
 
 ## Frontend ##
 
-doas pacman -S entr  # to watch source files
+sudo pacman -S entr  # to watch source files
 npm install -g --prefix ~/.node_modules esbuild # to bundle js
 
 # Listen for changes in js-src dir, automatically build minified bundle:


@@ 81,17 79,24 @@ Can be run with just `pytest`. It needs a pytaku.conf.json as well.
 - JavaScript: jshint, prettier
 
 ```sh
-doas pacman python-black python-isort flake8 prettier
+sudo pacman python-black python-isort flake8 prettier
 npm install -g --prefix ~/.node_modules jshint
 ```
 
 # Production
 
-This assumes Debian 11, consequently targeting python 3.9.
+I'm running my instance on Debian 11, but any unix-like environment with these
+should work:
+
+- python3.7+
+- apws (on Debian, simply install the `python3-apsw` package)
+- the rest are all pypi packages that should be automatically installed when
+  you run `pip install pytaku`
+
+The following is a step-by-step guide on Debian 11.
 
 ```sh
-# nodejs is used by cloudscraper to bypass Cloudflare
-sudo apt install nodejs python3-pip python3-apsw
+sudo apt install python3-pip python3-apsw
 pip3 install --user pytaku
 # now make sure ~/.local/bin is in your $PATH so pytaku commands are usable
 


@@ 118,12 123,13 @@ If you're exposing your instance to the internet, I don't have to remind you to
 properly set up a firewall and a TLS-terminating reverse proxy e.g.
 nginx/caddy, right?
 
-Alternatively, just setup [tailscale](https://tailscale.com/) and let them
-worry about access control and end-to-end encryption for you.
+Alternatively, just setup a personal [tailscale](https://tailscale.com/)
+network and let them worry about access control and end-to-end encryption for
+you.
 
 # LICENSE
 
-Copyright (C) 2021  Bùi Thành Nhân
+Copyright (C) 2021 Bùi Thành Nhân
 
 This program is free software: you can redistribute it and/or modify it under
 the terms of the GNU Affero General Public License version 3 as published by

M poetry.lock => poetry.lock +2 -34

@@ 97,19 97,6 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [[package]]
-name = "cloudscraper"
-version = "1.2.58"
-description = "A Python module to bypass Cloudflare's anti-bot page."
-category = "main"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-pyparsing = ">=2.4.7"
-requests = ">=2.9.2"
-requests-toolbelt = ">=0.9.1"
-
-[[package]]
 name = "colorama"
 version = "0.4.4"
 description = "Cross-platform colored terminal text."


@@ 415,7 402,7 @@ python-versions = ">=3.5"
 name = "pyparsing"
 version = "2.4.7"
 description = "Python parsing module"
-category = "main"
+category = "dev"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 


@@ 460,17 447,6 @@ security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
 socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
 
 [[package]]
-name = "requests-toolbelt"
-version = "0.9.1"
-description = "A utility belt for advanced users of python-requests"
-category = "main"
-optional = false
-python-versions = "*"
-
-[package.dependencies]
-requests = ">=2.0.1,<3.0.0"
-
-[[package]]
 name = "six"
 version = "1.16.0"
 description = "Python 2 and 3 compatibility utilities"


@@ 556,7 532,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "6ab6afb2d27321e79013421b33c1fa4011bdc5b33b04ffd20906b4a4122c0900"
+content-hash = "1753bef74f04a35a21914127fef75ab7bfc5c14e04c0c48613c0d637bcc6df91"
 
 [metadata.files]
 appnope = [


@@ 666,10 642,6 @@ click = [
     {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
     {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
 ]
-cloudscraper = [
-    {file = "cloudscraper-1.2.58-py2.py3-none-any.whl", hash = "sha256:674fd739f9412188aae8d6614e3e6316939fc0670ef5646abd3d316f1a59d3c2"},
-    {file = "cloudscraper-1.2.58.tar.gz", hash = "sha256:dda29028c5628b5ba3e4dc43816ed38fd46bd945ef938c420f185586a6d8dff2"},
-]
 colorama = [
     {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
     {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},


@@ 817,10 789,6 @@ requests = [
     {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
     {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
 ]
-requests-toolbelt = [
-    {file = "requests-toolbelt-0.9.1.tar.gz", hash = "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"},
-    {file = "requests_toolbelt-0.9.1-py2.py3-none-any.whl", hash = "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f"},
-]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},

M pyproject.toml => pyproject.toml +0 -1

@@ 27,7 27,6 @@ requests = "^2.24.0"
 goodconf = "^1.0.0"
 argon2-cffi = "^20.1.0"
 bbcode = "^1.1.0"
-cloudscraper = "^1.2.58"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.0.1"

M src/mangoapi/base_site.py => src/mangoapi/base_site.py +5 -6

@@ 1,7 1,6 @@
 from abc import ABC, abstractmethod
 from urllib.parse import urlparse
 
-import cloudscraper
 import requests
 
 from pytaku.conf import config


@@ 14,11 13,11 @@ from .exceptions import (
 
 
 def create_session():
-    return cloudscraper.create_scraper(
-        {
-            "mobile": False,
-        }
-    )
+    session = requests.Session()
+    session.headers[
+        "User-Agent"
+    ] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36"
+    return session
 
 
 class Site(ABC):