~nhanb/pytaku

e0389c9fe51d7d031040fbaa70ea330587aea75d — Bùi Thành Nhân a month ago dd995dd master 0.3.26
save both md & md@h links; fallback logic on FE

Preload logic is still wonky though: if fallback happens while
preloading, the alt urls (md@h ones) are cached but when user actually
navigates to next chapter, browser will try the default urls first, fail
again, then try alt urls again whose cache has expired (at least on
firefox when I tested it).
M README.md => README.md +7 -2
@@ 44,11 44,11 @@ pytaku-scheduler  # scheduled tasks e.g. update titles

## Frontend ##

sudo pacman -S entr  # to watch source files
doas pacman -S entr  # to watch source files
npm install -g --prefix ~/.node_modules esbuild # to bundle js

# Listen for changes in js-src dir, automatically build minified bundle:
find src/pytaku/js-src -name '*.js' | entr -r \
find src/pytaku/js-src -name '*.js' | entr -rc \
     esbuild src/pytaku/js-src/main.js \
     --bundle --sourcemap --minify \
     --outfile=src/pytaku/static/js/main.min.js


@@ 63,6 63,11 @@ Can be run with just `pytest`. It needs a pytaku.conf.json as well.
- Python: black, isort, flake8 without mccabe
- JavaScript: jshint, prettier

```sh
doas pacman python-black python-isort flake8 prettier
npm install -g --prefix ~/.node_modules jshint
```

# Production

```sh

M pyproject.toml => pyproject.toml +1 -1
@@ 1,6 1,6 @@
[tool.poetry]
name = "pytaku"
version = "0.3.25"
version = "0.3.26"
description = "Self-hostable web-based manga reader"
authors = ["Bùi Thành Nhân <hi@imnhan.com>"]
license = "AGPL-3.0-only"

M src/mangoapi/mangadex.py => src/mangoapi/mangadex.py +25 -12
@@ 52,24 52,37 @@ class Mangadex(Site):
        md_json = md_resp.json()
        assert md_json["status"] == "OK"

        # 'server' value points to a likely temporary MangaDex@Home instance, while
        # 'server_fallback' would be MD's own server e.g. s5.mangadex.org...
        # The latter may be down (like, literally at the time of writing), so for now
        # let's prioritize the MD@Home server.
        # I don't know how stable MD@Home links are, but it probably won't matter,
        # since `persistence.load_chapter()` will re-fetch if existing db record is more
        # than 1 day old anyway.
        # TODO: A more robust solution is to save both links to db, but I'm not in the
        # mood for it atm.
        server = md_json["server"] or md_json.get("server_fallback")
        img_path = f"{server}{md_json['hash']}"
        # 2 cases:
        # - If 'server_fallback' is absent, it means 'server' points to MD's own server
        #   e.g. s5.mangadex.org...
        # - Otherwise, 'server' points to a likely ephemeral MD@H node, while
        # 'server_fallback' now points to MD's own server.
        #
        # MD's own links apparently go dead sometimes, but MD@H links seem to expire
        # quickly all the time, so it's probably a good idea to store both anyway.

        server_fallback = md_json.get("server_fallback")
        if server_fallback:
            md_server = server_fallback
            mdah_server = md_json["server"]
        else:
            md_server = md_json["server"]
            mdah_server = None

        chapter = {
            "id": chapter_id,
            "title_id": str(md_json["manga_id"]),
            "site": "mangadex",
            "name": md_json["title"],
            "pages": [f"{img_path}/{page}" for page in md_json["page_array"]],
            "pages": [
                f"{md_server}{md_json['hash']}/{page}" for page in md_json["page_array"]
            ],
            "pages_alt": [
                f"{mdah_server}{md_json['hash']}/{page}"
                for page in md_json["page_array"]
            ]
            if mdah_server
            else [],
            "groups": _extract_groups(md_json),
            "is_webtoon": md_json["long_strip"] == 1,
            **_parse_chapter_number(md_json["chapter"]),

M src/mangoapi/mangasee.py => src/mangoapi/mangasee.py +1 -0
@@ 74,6 74,7 @@ class Mangasee(Site):
                _generate_img_src(img_server, title_id, chapter_data["Chapter"], p)
                for p in range(1, num_pages + 1)
            ],
            "pages_alt": [],
            "groups": [],
            "is_webtoon": False,
            **numbers,

M src/pytaku/database/migrations/latest_schema.sql => src/pytaku/database/migrations/latest_schema.sql +1 -1
@@ 63,7 63,7 @@ CREATE TABLE IF NOT EXISTS "chapter"(
    pages text,
    groups text,
    updated_at text default (datetime('now')),
    is_webtoon boolean,
    is_webtoon boolean, pages_alt text not null default '[]',

    foreign key (title_id, site) references title (id, site),
    unique(site, title_id, id)

A src/pytaku/database/migrations/m0007.sql => src/pytaku/database/migrations/m0007.sql +6 -0
@@ 0,0 1,6 @@
-- Add alternative page urls as backup because mangadex is flaky.
begin transaction;

alter table chapter add column pages_alt text not null default '[]';

commit;

M src/pytaku/js-src/routes/chapter.js => src/pytaku/js-src/routes/chapter.js +53 -9
@@ 35,6 35,29 @@ const ImgStatus = {
  FAILED: "failed",
};

function FallbackableImg(initialVNode) {
  let currentSrc;
  return {
    oninit: (vnode) => {
      currentSrc = vnode.attrs.src;
    },
    view: (vnode) => {
      return m("img", {
        src: currentSrc,
        style: vnode.attrs.style,
        onload: vnode.attrs.onload,
        onerror: (ev) => {
          if (currentSrc === vnode.attrs.src && vnode.attrs.altsrc !== null) {
            currentSrc = vnode.attrs.altsrc;
          } else {
            vnode.attrs.onerror(ev);
          }
        },
      });
    },
  };
}

function Chapter(initialVNode) {
  let isLoading = false;
  let chapter = {};


@@ 48,9 71,11 @@ function Chapter(initialVNode) {

  function loadNextPage() {
    if (pendingPages.length > 0) {
      let [src, altsrc] = pendingPages.splice(0, 1)[0];
      loadedPages.push({
        status: ImgStatus.LOADING,
        src: pendingPages.splice(0, 1)[0],
        src,
        altsrc,
      });
    } else if (chapter.next_chapter && nextChapterPromise === null) {
      /* Once all pages of this chapter have been loaded,


@@ 62,7 87,15 @@ function Chapter(initialVNode) {
        chapterId: chapter.next_chapter.id,
      }).then((nextChapter) => {
        console.log("Preloading next chapter:", fullChapterName(nextChapter));
        nextChapterPendingPages = nextChapter.pages.slice();
        if (nextChapter.pages_alt.length > 0) {
          nextChapterPendingPages = nextChapter.pages.map((page, i) => {
            return [page, nextChapter.pages_alt[i]];
          });
        } else {
          nextChapterPendingPages = nextChapter.pages.map((page) => {
            return [page, null];
          });
        }
        // Apparently preloading one at a time was too slow so let's go with 2.
        preloadNextChapterPage();
        preloadNextChapterPage();


@@ 73,7 106,8 @@ function Chapter(initialVNode) {
  function preloadNextChapterPage() {
    if (nextChapterPendingPages !== null) {
      if (nextChapterPendingPages.length > 0) {
        nextChapterLoadedPages.push(nextChapterPendingPages.splice(0, 1)[0]);
        const [src, altsrc] = nextChapterPendingPages.splice(0, 1)[0];
        nextChapterLoadedPages.push({ src, altsrc });
      }
    }
  }


@@ 96,8 130,16 @@ function Chapter(initialVNode) {
          chapter = resp;
          document.title = fullChapterName(chapter);

          // Clone array here to avoid mutating the model
          pendingPages = chapter.pages.slice();
          // "zip" pages & pages_alt into pendingPages
          if (chapter.pages_alt.length > 0) {
            pendingPages = chapter.pages.map((page, i) => {
              return [page, chapter.pages_alt[i]];
            });
          } else {
            pendingPages = chapter.pages.map((page) => {
              return [page, null];
            });
          }

          // start loading pages, 3 at a time:
          loadNextPage();


@@ 184,8 226,9 @@ function Chapter(initialVNode) {
          [
            loadedPages.map((page, pageIndex) =>
              m("div", { key: page.src }, [
                m("img", {
                m(FallbackableImg, {
                  src: page.src,
                  altsrc: page.altsrc,
                  style: {
                    display:
                      page.status === ImgStatus.SUCCEEDED ? "block" : "none",


@@ 211,16 254,17 @@ function Chapter(initialVNode) {
                  : null,
              ])
            ),
            pendingPages.map((page) => m(PendingPlaceholder)),
            pendingPages.map(() => m(PendingPlaceholder)),
          ]
        ),
        buttons,
        nextChapterLoadedPages.map((page) =>
          m("img.chapter--preloader", {
          m(FallbackableImg, {
            style: { display: "none" },
            onload: preloadNextChapterPage,
            onerror: preloadNextChapterPage,
            src: page,
            src: page.src,
            altsrc: page.altsrc,
          })
        ),
      ]);

M src/pytaku/main.py => src/pytaku/main.py +1 -0
@@ 283,6 283,7 @@ def api_chapter(site, title_id, chapter_id):

    if site in ("mangadex", "mangasee"):
        chapter["pages"] = [proxied(p) for p in chapter["pages"]]
        chapter["pages_alt"] = [proxied(p) for p in chapter["pages_alt"]]

    # YIIIIKES
    title = load_title(site, title_id)

M src/pytaku/persistence.py => src/pytaku/persistence.py +6 -1
@@ 105,6 105,7 @@ def save_chapter(chapter):
        num_minor,
        name,
        pages,
        pages_alt,
        groups,
        is_webtoon
    ) VALUES (


@@ 115,6 116,7 @@ def save_chapter(chapter):
        :num_minor,
        :name,
        :pages,
        :pages_alt,
        :groups,
        :is_webtoon
    ) ON CONFLICT (id, title_id, site) DO UPDATE SET


@@ 122,6 124,7 @@ def save_chapter(chapter):
        num_minor=excluded.num_minor,
        name=excluded.name,
        pages=excluded.pages,
        pages_alt=excluded.pages_alt,
        groups=excluded.groups,
        is_webtoon=excluded.is_webtoon,
        updated_at=datetime('now')


@@ 135,6 138,7 @@ def save_chapter(chapter):
            "num_minor": chapter.get("num_minor"),
            "name": chapter["name"],
            "pages": json.dumps(chapter["pages"]),
            "pages_alt": json.dumps(chapter["pages_alt"]),
            "groups": json.dumps(chapter["groups"]),
            "is_webtoon": chapter["is_webtoon"],
        },


@@ 145,7 149,7 @@ def load_chapter(site, title_id, chapter_id, ignore_old=True):
    updated_at = "datetime('now', '-1 days')" if ignore_old else "'1980-01-01'"
    result = run_sql(
        f"""
        SELECT id, title_id, site, num_major, num_minor, name, pages, groups, is_webtoon
        SELECT id, title_id, site, num_major, num_minor, name, pages, pages_alt, groups, is_webtoon
        FROM chapter
        WHERE site=? AND title_id=? AND id=? AND updated_at > {updated_at};
        """,


@@ 158,6 162,7 @@ def load_chapter(site, title_id, chapter_id, ignore_old=True):
    else:
        chapter = result[0]
        chapter["pages"] = json.loads(chapter["pages"])
        chapter["pages_alt"] = json.loads(chapter["pages_alt"])
        chapter["groups"] = json.loads(chapter["groups"])
        return chapter


M tests/mangoapi/test_mangadex.py => tests/mangoapi/test_mangadex.py +2 -0
@@ 85,6 85,7 @@ def test_get_title():
def test_get_chapter():
    chap = Mangadex().get_chapter("doesn't matter", "696882")
    pages = chap.pop("pages")
    pages_alt = chap.pop("pages_alt")
    assert chap == {
        "id": "696882",
        "title_id": "12088",


@@ 97,6 98,7 @@ def test_get_chapter():
        "num_minor": 5,
    }
    assert len(pages) == 16
    assert len(pages_alt) == 16


def test_search():

M tests/mangoapi/test_mangasee.py => tests/mangoapi/test_mangasee.py +2 -0
@@ 29,6 29,7 @@ def test_get_title():
def test_get_chapter():
    chapter = Mangasee().get_chapter("Yu-Yu-Hakusho", "63.5")
    pages = chapter.pop("pages")
    pages_alt = chapter.pop("pages_alt")
    assert chapter == {
        "groups": [],
        "id": "63.5",


@@ 42,6 43,7 @@ def test_get_chapter():
    }
    assert pages[0] == "https://s1.mangabeast01.com/manga/Yu-Yu-Hakusho/0063.5-001.png"
    assert pages[-1] == "https://s1.mangabeast01.com/manga/Yu-Yu-Hakusho/0063.5-031.png"
    assert pages_alt == []


def test_search_title():