~reesmichael1/roman

6c19394d408416c00b4d95f336a0f24b4487b879 — Michael Rees 8 months ago 9603559
Initial attempt at downloading feeds in parallel
3 files changed, 16 insertions(+), 6 deletions(-)

M src/romanpkg/feeds.nim
M src/romanpkg/htmlextractor.nim
M src/romanpkg/main.nim
M src/romanpkg/feeds.nim => src/romanpkg/feeds.nim +9 -3
@@ 1,6 1,7 @@
import sequtils
import tables
import terminal
import threadpool

import fab
import FeedNim


@@ 50,12 51,14 @@ proc displayFeed*(feed: var Feed) {.raises: [RomanError].} =
    raise newException(RomanError, "could not set terminal style: " & e.msg)


proc getFeed*(url: string): Feed {.raises: [RomanError].} =
proc getFeed*(url: string): Feed {.raises: [RomanError], thread.} =
  try:
    let rssFeed = FeedNim.getRSS(url)
    result.title = rssFeed.title
    result.posts = map(rssFeed.items,
      proc (i: RSSItem): Post = postFromRSSItem(i))
    for item in rssFeed.items:
      result.posts.add(postFromRSSItem(item))
    # result.posts = map(rssFeed.items,
    #   proc (i: RSSItem): Post = postFromRSSItem(i))
    result.updateUnread()
  except ValueError:
    raise newException(RomanError, url & " is not a valid URL")


@@ 63,3 66,6 @@ proc getFeed*(url: string): Feed {.raises: [RomanError].} =
    let msg = getCurrentExceptionMsg()
    raise newException(RomanError,
      "error while accessing " & url & ": " & msg)


# proc getFeedAsync*(url: string): Future[Feed] {.async, raises: [].} =

M src/romanpkg/htmlextractor.nim => src/romanpkg/htmlextractor.nim +3 -2
@@ 8,7 8,8 @@ import errors

proc extractBody*(body: string): string {.raises: [RomanError].} =
  try:
    let html2text = pyImport("html2text").HTML2Text()
    result = html2text.handle(body).to(string)
    result = body
    # let html2text = pyImport("html2text").HTML2Text()
    # result = html2text.handle(body).to(string)
  except:
    raise newException(RomanError, "could not use Python module html2text")

M src/romanpkg/main.nim => src/romanpkg/main.nim +4 -1
@@ 1,6 1,7 @@
import os
import sequtils
import tables
import threadpool

import errors
import feeds


@@ 32,7 33,9 @@ proc runMainPath() {.raises: [RomanError].} =
  elif subs.len == 1:
    feed = getFeed(subs[0].url)
  else:
    let feeds = map(subs, proc(s: Subscription): Feed = getFeed(s.url))
    let feedResults = map(subs, proc(s: Subscription): FlowVar[
        Feed] = spawn getFeed(s.url))
    let feeds = map(feedResults, proc(fv: FlowVar[Feed]): Feed = ^fv)
    feed = chooseFeed(feeds)

  displayFeed(feed)