1cbed277a826e24c14f387da5c95683f330caab5 — Michael Rees 4 months ago fff97e0
Load feeds in parallel at the start

This (finally!) closes #11. Now that we explicitly tell the compiler
that html2text.handle is GC-safe, we can call it from a parallel block,
which allows us to parallelize the feed parsing. It turns out that it's
actually only a small time improvement, but it's still nice.
1 files changed, 30 insertions(+), 11 deletions(-)

M src/romanpkg/feeds.nim
M src/romanpkg/feeds.nim => src/romanpkg/feeds.nim +30 -11
@@ 1,3 1,7 @@
when defined(internalRenderer):
  {.experimental: "parallel".}
  import threadpool

import asyncdispatch
import httpclient
import options

@@ 21,8 25,8 @@ import termask
from types import Feed, FeedKind, Post, Subscription

let atomNames = ["index.atom", "feed.atom", "atom.xml"]
let rssNames = ["index.rss", "feed.rss", "rss.xml"]
const atomNames = ["index.atom", "feed.atom", "atom.xml"]
const rssNames = ["index.rss", "feed.rss", "rss.xml"]

proc updateUnread*(feed: var Feed) {.raises: [].} =

@@ 109,8 113,9 @@ proc displayFeed*(feed: var Feed) {.raises: [RomanError, InterruptError].} =
    raise newException(RomanError, "could not set terminal style: " & e.msg)

proc buildFeedFromContentAndSub(content: string, sub: Subscription): Feed {.
proc buildFeedFromContentAndSub(content: string, sub: Subscription): ref Feed {.
    raises: [RomanError].} =
  result = new Feed
    var feedKind = sub.feedKind
    if feedKind == Unknown:

@@ 136,7 141,7 @@ proc buildFeedFromContentAndSub(content: string, sub: Subscription): Feed {.
      raise newException(RomanError,
        "could not identify feed as RSS or Atom, please use --type option")
    result.kind = feedKind
  except ValueError:
    raise newException(RomanError, sub.url & " is not a valid URL")

@@ 149,26 154,40 @@ proc getFeed*(sub: Subscription): Feed {.raises: [RomanError].} =
    var client = newHttpClient()
    let content = client.getContent(sub.url)
    result = buildFeedFromContentAndSub(content, sub)
    result = buildFeedFromContentAndSub(content, sub)[]
  except Exception as e:
    raise newException(RomanError, e.msg)

proc asyncFeedsLoader(subs: seq[Subscription]): Future[seq[Feed]] {.async.} =
proc asyncFeedsLoader(subs: seq[Subscription]): Future[seq[string]] {.async.} =
  var futures = newSeq[Future[string]](subs.len)
  result = newSeq[Feed](subs.len)
  result = newSeq[string](subs.len)
  for ix, sub in subs:
    var client = newAsyncHttpClient()
    futures[ix] = client.getContent(sub.url)

  let contents = await all(futures)
  for ix, content in contents:
    result[ix] = buildFeedFromContentAndSub(content, subs[ix])
  result = await all(futures)

proc getFeeds*(subs: seq[Subscription]): seq[Feed] {.raises: [RomanError].} =
  result = newSeq[Feed](subs.len)
  var contents: seq[string]
    result = waitFor asyncFeedsLoader(subs)
    contents = waitFor asyncFeedsLoader(subs)
    when defined(internalRenderer):
      var responses = newSeq[FlowVar[ref Feed]](subs.len)
        for ix, content in contents:
          responses[ix] = spawn buildFeedFromContentAndSub(content, subs[ix])

      for ix, response in responses:
        result[ix] = (^response)[]

      for ix, content in contents:
        result[ix] = buildFeedFromContentAndSub(content, subs[ix])[]

    raise newException(RomanError, "error in loading feeds: " &