~tieong/download-course

a31f871f402b4bf875f585abb0634a9a3d856751 — Thomas Ieong 2 years ago 570ab6a
Added retries mechanism

Sometimes the server returns a 503, this Retry object ensures that we
don't give up immediately.

TODO: Wrap the GET/POST requests in try except block in case it fails even
with the retries, so that the script doesn't crash.
1 files changed, 8 insertions(+), 0 deletions(-)

M download_course/download_course.py
M download_course/download_course.py => download_course/download_course.py +8 -0
@@ 12,6 12,7 @@ from typing import List
from bs4 import BeautifulSoup
from bs4.element import ResultSet, Tag
import requests
from requests.adapters import HTTPAdapter, Retry

from download_course.utils.utils import (
    get_documents_element,


@@ 491,6 492,13 @@ def main() -> None:
        logging.getLogger().setLevel(logging.DEBUG)

    REPLACE_ME_session = requests.Session()
    retries = Retry(
        total=10,
        backoff_factor=10,
        status_forcelist=[500, 502, 503, 504]
    )
    REPLACE_ME_session.mount('https://', HTTPAdapter(max_retries=retries))

    with REPLACE_ME_session:
        download_courses(args, REPLACE_ME_session, download_dir_path)