~nhanb/pytaku

0907d0bf67a6eb5ca5946bc0dbf7f80b5644a74f — Nhân 1 year, 4 months ago c35ec01
schduler: ignore 404 errors

May consider automatically clean up 404 titles as well.
3 files changed, 16 insertions(+), 2 deletions(-)

M src/mangoapi/base_site.py
M src/mangoapi/exceptions.py
M src/pytaku/scheduler.py
M src/mangoapi/base_site.py => src/mangoapi/base_site.py +3 -0
@@ 7,6 7,7 @@ from pytaku.conf import config

from .exceptions import (
    SourceSite5xxError,
    SourceSite404Error,
    SourceSiteTimeoutError,
    SourceSiteUnexpectedError,
)


@@ 79,6 80,8 @@ class Site(ABC):

        if 500 <= resp.status_code <= 599:
            raise SourceSite5xxError(url, resp.status_code, resp.text)
        elif resp.status_code == 404:
            raise SourceSite404Error(url, resp.text)
        elif resp.status_code != 200:
            raise SourceSiteUnexpectedError(url, resp.status_code, resp.text)


M src/mangoapi/exceptions.py => src/mangoapi/exceptions.py +5 -0
@@ 13,5 13,10 @@ class SourceSite5xxError(SourceSiteResponseError):
    pass


class SourceSite404Error(SourceSiteResponseError):
    def __init__(self, url, response_text):
        super().__init__(url, 404, response_text)


class SourceSiteUnexpectedError(SourceSiteResponseError):
    pass

M src/pytaku/scheduler.py => src/pytaku/scheduler.py +8 -2
@@ 7,7 7,7 @@ from pathlib import Path

from requests.exceptions import ReadTimeout

from mangoapi.exceptions import SourceSite5xxError
from mangoapi.exceptions import SourceSite5xxError, SourceSite404Error

from .conf import config
from .persistence import delete_expired_tokens, find_outdated_titles, save_title


@@ 77,7 77,13 @@ class UpdateOutdatedTitles(Worker):
                print(" done")
                if title["site"] == "mangasee":
                    time.sleep(2)
            except (SourceSite5xxError, ReadTimeout, JSONDecodeError) as e:
            except (
                # TODO: on 404, maybe delete title that's no longer on source site?
                SourceSite404Error,
                SourceSite5xxError,
                ReadTimeout,
                JSONDecodeError,
            ) as e:
                print(" skipped because of server error:", e.__class__.__name__, str(e))