~kungtotte/dtt

4c4eaef23e1be76af2755800eee8a5681752efff — Thomas Landin 6 months ago b4f1639
Optimize buildCmd to use fewer loops

This is a big change that optimizes away the multiple loops to scan for
blog posts, find links to pages, and render pages out. Now it does one
major loop collecting everything then one loop to render and write pages
out that goes much faster since it's doing fewer things.

Next up is refactoring the code to reduce duplication and complexity.
2 files changed, 105 insertions(+), 64 deletions(-)

M src/dtt.nim
M src/utils.nim
M src/dtt.nim => src/dtt.nim +63 -44
@@ 1,5 1,5 @@
import os, strutils, strformat
import parsecfg
import unicode

import docopt
import mustache


@@ 17,7 17,7 @@ const
                  fpGRoupRead, fpOthersExec, fpOthersRead}

let
  version = "0.2.0"
  version = "0.2.5"

type
  DttError {.pure.} = enum


@@ 28,6 28,14 @@ type
    RemoveDirectoryFailed,
    UnknownError

  PageMeta = tuple
    filename: string
    abs_path: string
    out_path: string
    rel_path: string
    tmpl: string
    title: string

proc initCmd(dir: string, force: bool = false) =
  let wd = if dir == "nil": getCurrentDir() else: getCurrentDir() / dir
  if not existsDir(wd):


@@ 90,53 98,63 @@ proc cleanCmd(all: bool = false, force: bool = false) =
    stderr.writeLine(fmt"{bin}: {wd} does not exist.")
    quit(ord(DttError.DirectoryNotFound))

proc buildCmd() =
  let (config, working_dir) = try:
proc buildCmd(posts_per_page: int = 5) =
  # TODO: Rationalized the config.cfg away! Scan for directory structure instead
  let (_, working_dir) = try:
    findConfigFile(os.getCurrentDir() / "config.cfg")
  except IOError:
    stderr.writeLine(fmt"{bin}: Could not find a config.cfg file. Is this a dtt directory?")
    quit(ord(DttError.NotADttDirectory))
  let
    condir = working_dir / content_dir
    tmpldir = working_dir / template_dir
    outdir = working_dir / output_dir
    content_dir = working_dir / content_dir
    tmpl_dir = working_dir / template_dir
    output_dir = working_dir / output_dir

  if not existsDir(output_dir):
    createDir(output_dir)

  var
    mcontext = newContext(searchDirs = @[tmpldir])

  if not existsDir(outdir):
    createDir(outdir)

  loadMetaData(config, mcontext)
  let blogs = findBlogPosts(condir, tmpldir)
  mcontext["links"] = findLinks(condir, excludes=blogs)

  for f in walkDirRec(condir, {pcFile, pcDir}, relative = true):
    case getFileInfo(condir / f).kind:
      of pcDir:
        createDir(outdir / f)
      of pcFile:
        let
          (dir, fname, ext) = splitFile(f)
          savepath = joinPath(outdir, dir)
        if ext == ".md":
          let tmpl = findTemplate(fname, tmpldir)
          if blogs.len > 0:
            let
              cfg_bcount = config.getSectionValue("Site", "blogs_per_page").parseInt
              bcount = min(blogs.len - 1, cfg_bcount)
            mcontext["blog_posts"] = blogs[0..bcount]
          let
            output  = mcontext.renderTemplate(condir / f, tmpl, tmpldir = tmpldir)
            outfile = savepath / fname & ".html"
          writeFile(outfile, output)
          setFilePermissions(outfile, outFilePerms)
        else:
          let outfile = savepath / fname & ext
          copyFile(condir / f, outfile)
          setFilePermissions(outfile, outFilePerms)
    pages: seq[PageMeta]
    posts: seq[string]
    links: seq[Table[string, string]]
  for f in walkDirRec(content_dir, relative = true):
    let
      (dir, name, ext) = splitFile(f)
      savepath = output_dir / dir
    createDir(savepath)
    if ext != ".md":
      # Any non-Markdown files are just copied straight across
      # This lets people use static assets and also hand-craft
      # html pages if they want to.
      let output_file = savepath / name & ext
      copyFile(content_dir / f, output_file)
      setFilePermissions(output_file, outFilePerms)
    else:
      if isBlog(dir, name):
        let rendered = renderBlog(content_dir / f, tmpl_dir)
        posts.add(rendered)
        writeFile(savepath / name & ".html", rendered)
      else:
        stderr.writeLine(fmt"Unknown error occurred: {f} is neither file nor directory")
        quit(ord(DttError.UnknownError))
        var page: PageMeta
        page.title = name.title()
        page.filename = name & ".html"
        page.abs_path = content_dir / f
        page.rel_path = dir / name & ".html"
        page.out_path = savepath
        page.tmpl = if existsFile(tmpl_dir / name & ".mustache"): name else: "page"
        links.add({"target": page.rel_path, "title": page.title}.toTable)
        pages.add(page)

  let
    num_blogs = min(posts.len - 1, posts_per_page)
  for page in pages:
    var context = newContext(searchDirs = @[tmpl_dir])
    context["blog_posts"] = posts[0..num_blogs]
    context["links"] = links
    let
      rendered = context.renderTemplate(page.abs_path, page.tmpl, tmpl_dir)
      output_file = page.out_path / page.filename
    writeFile(output_file, rendered)


let doc = """


@@ 150,7 168,7 @@ content from any host.

usage:
  dtt init [options] [DIR]
  dtt build [options]
  dtt build [options] [POSTS]
  dtt clean [options]

Options:


@@ 167,4 185,5 @@ when isMainModule:
  elif args["clean"]:
    cleanCmd(args["--all"], args["--force"])
  elif args["build"]:
    buildCmd()
    let posts = if args["POSTS"].kind == vkNone: 5 else: parseInt($args["POSTS"])
    buildCmd(posts)

M src/utils.nim => src/utils.nim +42 -20
@@ 17,30 17,13 @@ proc dateSort(x, y: Table[string, string]): int =
  else:
    -1

proc loadMetaData*(cfgdict: Config, mcontext: var Context) =
  # TODO: Make dtt move backwards up the hierarchy to look for config.cfg
  # so we don't error out if someone happens to run `dtt build` inside the
  # content  directory instead of the root dtt directory.

  # Load meta-data from config file into the supplied
  # mustache context
  mcontext["charset"] = cfgdict.getSectionValue("Site", "charset")
  mcontext["language"] = cfgdict.getSectionValue("Site","language")
  mcontext["page_title"] = cfgdict.getSectionValue("Site","title")
  mcontext["content_license"] = cfgdict.getSectionValue("Site","license")
  mcontext["content_license_url"] = cfgdict.getSectionValue("Site","license_url")
  mcontext["email"] = cfgdict.getSectionValue("Author", "email")
  mcontext["author"] = cfgdict.getSectionValue("Author", "name")

proc renderTemplate*(context: Context, file: string,
                    tmpl: string = "page",
                    tmpldir: string = "templates"): string =
  let
    tpl = readFile(tmpldir / tmpl & ".mustache")
    (_,_,ext) = splitFile(file)
  if ext == ".md":
    context["content"] = markdown(readFile(file))
    result = tpl.render(context)
  context["content"] = markdown(readFile(file))
  result = tpl.render(context)


proc findLinks*(content_dir: string, excludes: seq[Table[string,string]]): seq[Table[string, string]] =


@@ 71,7 54,38 @@ proc getSlug*(filename: string, hasYMD: bool): string =
    result = result[11..^1]
  result = result.title().multiReplace(("-", " "), ("_", " "))


proc formatPaddedDate(year, month, day: int): string =
  # Return a zero-padded date string in the format YYYY-MM-DD
  # fmt uses parseInt behind the scenes which will kill leading
  # zeroes so 2020-01-09 becomes 2020-1-9 when using fmt"" which
  # is not what we want.
  let
    y = intToStr(year)
    m = intToStr(month, 2)
    d = intToStr(day, 2)
  result = fmt"{y}-{m}-{d}"

proc findSlugAndDate(name: string): tuple[slug, date: string] =
  # Extract the (optional) YYYY-MM-DD date portion of the filename
  # as well as the remainder of the string properly formatted as a title
  # with any separating characters replaced by spaces.
  result.slug = name
  result.date = ""
  var
    y, m, d:  int
  if scanf(name, "$i-$i-$i", y, m, d):
    result.slug = name[11..^1]
    result.date = formatPaddedDate(y, m, d)
  result.slug = result.slug.title().multiReplace(("-", " "), ("_", " "))

proc renderBlog*(file: string, tmpl_dir: string): string =
  let
    (_, name, _) = splitFile(file)
    header = findSlugAndDate(name)
  var context = newContext(searchDirs = @[tmpl_dir])
  context["slug"] = header.slug
  context["date"] = header.date
  result = context.renderTemplate(file, "post", tmpl_dir)
proc getBlogDate*(file: string, datestring: tuple[year, month, day: int]): string =
  if datestring.year > 0:
    let


@@ 87,6 101,14 @@ proc getBlogDate*(file: string, datestring: tuple[year, month, day: int]): strin
    # using the yyyy-MM-dd-slug.md format.
    result = getCreationTime(file).format("yyyy-MM-dd")

proc isBlog*(path, name: string): bool =
  var
    y, m, d: int
  if path in blog_dirs or scanf(name, "$i-$i-$i", y, m, d):
    result = true
  else:
    result = false

proc findBlogPosts*(directory: string, tmpldir: string): seq[Table[string, string]] =
  for f in walkDirRec(directory):
    let (dir, name, ext) = splitFile(f)