~subsetpark/bagatto

501e43f1910759db8443d9a5d66d97497528f439 — Zach Smith 9 months ago 01c5c03
jfmt
M bagatto.janet => bagatto.janet +25 -25
@@ 76,13 76,13 @@

(defn- adjust-for-display [date]
  (-> (fn [[k v]]
       (if (and (number? v)
                (index-of k [:month-day :month :year-day :week-day]))
         [k (inc v)]
         [k v]))
     (mapcat (pairs date))
     (splice)
     (struct)))
        (if (and (number? v)
                 (index-of k [:month-day :month :year-day :week-day]))
          [k (inc v)]
          [k v]))
      (mapcat (pairs date))
      (splice)
      (struct)))

(defn datestr->secs
  ```


@@ 101,8 101,8 @@
  ```
  [datestr &opt for-display?]
  (let [parsed (-> datestr
                  (datestr->secs)
                  (os/date))]
                   (datestr->secs)
                   (os/date))]
    (if for-display? (adjust-for-display parsed) parsed)))

#


@@ 116,8 116,8 @@
  (try
    (sh/glob pattern :x)
    ([err fib]
     (print "[" pattern "] " err)
     @[])))
      (print "[" pattern "] " err)
      @[])))

(defn *
  ```


@@ 192,7 192,7 @@

  (def render-context @{:current-template template})
  (set-error-context! :render-context render-context)
  

  (comptime (def- bufsize (core-* 12 1024)))
  (let [res (buffer/new bufsize)
        env (require template)


@@ 224,7 224,7 @@
  (see `bagatto/slugify` for more details.)
  ```
  [item key]
  (string (slugify (item key)))) 
  (string (slugify (item key))))

#
# FOFs


@@ 264,7 264,7 @@
  [path]
  (fn [site item] (get-in item path)))

(defn %p 
(defn %p
  ````
  Simple DSL for generating paths. 



@@ 290,9 290,9 @@
  [& elements]
  (fn [site &opt item]
    (default item {})
    

    (def path-parts @[])
    

    (var item-key? false)
    (var site-key? false)
    (var push-to-last? false)


@@ 302,19 302,19 @@
      (set site-key? false))

    (defn reset-push! [] (set push-to-last? false))
    

    (defn handle-element [x]
      (cond
        (= x '%) (set push-to-last? true)
        (= x '%i) (set item-key? true)
        (= x '%s) (set site-key? true)
        (let [path-component (cond item-key? (do (reset-flags!) (slug-from-attr item x))
                                   site-key? (do (reset-flags!) (slug-from-attr site x))
                                   x)]
                               site-key? (do (reset-flags!) (slug-from-attr site x))
                               x)]
          (if push-to-last?
            (do (reset-push!) (buffer/push-string (last path-parts) path-component))
            (array/push path-parts (buffer path-component))))))
    

    (each element elements (handle-element element))
    (path/join (splice path-parts))))



@@ 326,7 326,7 @@
  [base item key]
  (path/join base (path/basename (item key))))

(defn path-copier 
(defn path-copier
  ````
  Return a function that will generate a new path with the same
  base, from the same key, for any item.


@@ 351,7 351,7 @@
  ```
  Return a sorter function that, given a list of items, sorts
  according to the items' values at the specified key.
  ``` 
  ```
  [key &opt descending?]
  (def by (fn [x y] ((if descending? > <) (x key) (y key))))
  (fn [items] (sort items by)))


@@ 404,8 404,8 @@
  ```
  [writer-specs output-dir]
  (-> writer-specs
     (threads/demand-pipeline writers/writer-init
                              (writers/handle-writes output-dir))))
      (threads/demand-pipeline writers/writer-init
                               (writers/handle-writes output-dir))))

#
# TEMPLATE


@@ 453,7 453,7 @@
  [md &keys {:smart smart}]
  (multimarkdown/snippet md smart))

(defn format 
(defn format
  ```
  A simple wrapper around `string/format` to ease development. If one
  of `xs` is nil, it will output an empty string rather than crashing

M demo/index.janet => demo/index.janet +2 -2
@@ 97,7 97,7 @@
# Here we use the bagatto/%p function to generate a path function for
# us. It exposes a simple DSL and returns a function that will take
# the site data and a post, and return the path for that post.
(def make-post-path (bagatto/%p "posts" '%i :title '% ".html" ))
(def make-post-path (bagatto/%p "posts" '%i :title '% ".html"))

# We can use another generator function, bagatto/path-copier, in the
# case of static paths. This returns a function which will take an


@@ 119,7 119,7 @@
# post and call `bagatto/render` on the specified template, with the
# site and the post as arguments.
(def render-post (bagatto/renderer "templates/post" {:root "../"}))
                

# The Posts index is generated from the site data and lists out all
# the posts. Therefore, it isn't rendered out of a specific source
# file. We will only call it once, with the site data as the only

M main.janet => main.janet +2 -3
@@ 15,7 15,7 @@
(defn- index-value
  [env sym index]
  (try ((env sym) :value)
       ([err fib] (error/eval-error sym index))))
    ([err fib] (error/eval-error sym index))))

(def argparse-params ["A transparent, extensible static site generator."
                      "repl" {:kind :flag


@@ 44,9 44,8 @@

        (def data (let [data-spec (index-value env 'data index)]
                    (core/load-data data-spec)))
        

        (def writer-specs (let [site-spec (index-value env 'site index)]
                            (core/produce-writer-specs site-spec data env bagatto)))

        
        (core/evaluate-writer-specs env writer-specs)))))

M project.janet => project.janet +4 -3
@@ 10,8 10,9 @@
                 "https://github.com/janet-lang/argparse.git"])

(declare-executable
 :name "bag"
 :entry "main.janet")
  :name "bag"
  :entry "main.janet"
  :install true)

(declare-source
 :source ["bagatto.janet"])
  :source ["bagatto.janet"])

M src/core.janet => src/core.janet +69 -71
@@ 37,30 37,30 @@
  function that will be called on each file-contents.
  ```
  [data-spec]
  

  (let [data-pairs (pairs data-spec)
        jobs (seq [[spec-name spec] :in data-pairs]

                  (threads/print "Reading " spec-name " data spec...")
                        
                  (setdyn :error-context {:spec-name spec-name})

                  (let [with-defaults (set-defaults spec)
                        transform-f (or (spec :transform) identity)]
                    (match with-defaults
                      ({:src loader :attrs parser} (function? loader))
                      (loaders/from-file-spec-loader spec-name loader parser transform-f)
                                
                      ({:src path :attrs parser} (string? path))
                      (loaders/from-path-loader spec-name path parser)
                                
                      {:attrs attrs}
                      (loaders/bare-attr-loader spec-name attrs)
                                
                      _ (error/data-error with-defaults))))]
               (threads/print "Reading " spec-name " data spec...")

               (setdyn :error-context {:spec-name spec-name})

               (let [with-defaults (set-defaults spec)
                     transform-f (or (spec :transform) identity)]
                 (match with-defaults
                   ({:src loader :attrs parser} (function? loader))
                   (loaders/from-file-spec-loader spec-name loader parser transform-f)

                   ({:src path :attrs parser} (string? path))
                   (loaders/from-path-loader spec-name path parser)

                   {:attrs attrs}
                   (loaders/bare-attr-loader spec-name attrs)

                   _ (error/data-error with-defaults))))]
    (threads/distribute jobs)))

(defn produce-writer-specs 
(defn produce-writer-specs
  ```
  Second phase of main business logic. `site` contains a specification
  for generating a website and `data` is all the source data we have


@@ 70,60 70,58 @@
  [site data bagatto env]
  (let [site-pairs (pairs site)
        jobs (seq [[spec-name spec] :in site-pairs]
               
                  (threads/print "Reading " spec-name " site spec...")
               
                  (let [with-defaults (set-defaults spec)
                        filter (spec :filter)]
                    (default filter (fn [_site _item] true))
      
      
                    (match with-defaults
                      {:each site-selector
                       :dest path-generator
                       :out renderer}
                      (generators/render-each-generator bagatto
                                                        env
                                                        data
                                                        spec-name
                                                        filter
                                                        site-selector
                                                        path-generator
                                                        renderer)
        
                      {:each site-selector
                       :dest path-generator}
                      (generators/copy-each-generator bagatto
                                                      env
                                                      data
                                                      spec-name
                                                      filter
                                                      site-selector
                                                      path-generator)
        
                      {:dest path-generator
                       :out renderer}
                      (generators/render-generator bagatto

               (threads/print "Reading " spec-name " site spec...")

               (let [with-defaults (set-defaults spec)
                     filter (spec :filter)]
                 (default filter (fn [_site _item] true))

                 (match with-defaults
                   {:each site-selector
                    :dest path-generator
                    :out renderer}
                   (generators/render-each-generator bagatto
                                                     env
                                                     data
                                                     spec-name
                                                     filter
                                                     site-selector
                                                     path-generator
                                                     renderer)

                   {:each site-selector
                    :dest path-generator}
                   (generators/copy-each-generator bagatto
                                                   env
                                                   data
                                                   spec-name
                                                   filter
                                                   site-selector
                                                   path-generator)

                   {:dest path-generator
                    :out renderer}
                   (generators/render-generator bagatto
                                                env
                                                data
                                                spec-name
                                                path-generator
                                                renderer)

                   # TODO: When is this necessary? 
                   {:some site-selector
                    :dest path-generator}
                   (generators/copy-some-generator bagatto
                                                   env
                                                   data
                                                   spec-name
                                                   path-generator
                                                   renderer)
        
                      # TODO: When is this necessary? 
                      {:some site-selector
                       :dest path-generator}
                      (generators/copy-some-generator bagatto
                                                      env
                                                      data
                                                      spec-name
                                                      site-selector
                                                      path-generator)
                      

                      _ (error/site-error with-defaults))))]
        
    (let [segments  (-> (threads/distribute jobs) (values))]
                                                   site-selector
                                                   path-generator)

                   _ (error/site-error with-defaults))))]

    (let [segments (-> (threads/distribute jobs) (values))]
      (array/concat ;segments))))

(defn evaluate-writer-specs


@@ 132,7 130,7 @@
  them into new files.
  ```
  [env writer-specs]
  

  (def output-dir (env :bagatto-output-dir))

  (threads/demand-pipeline writer-specs

M src/env.janet => src/env.janet +1 -1
@@ 5,7 5,7 @@
  (match (os/getenv "JANET_PATH")
    nil :ok
    janet-path (put root-env :syspath janet-path))
  (merge-into root-env bagatto) )
  (merge-into root-env bagatto))

(defn prepare-env-with-index! [env]
  # Monkey-patch the temple environment with the functions defined

M src/error.janet => src/error.janet +22 -22
@@ 8,23 8,23 @@
  (let [cxt (error-context :render-context)]
    (if (any? [cxt])
      (string/format
       ```
        ```
       
Current render context:
%q
       ```
       cxt)
        cxt)
      "")))

(defn eval-error
  [sym filename]
  (error
   (string "Expected symbol " sym "; couldn't be found after evaluating " filename)))
    (string "Expected symbol " sym "; couldn't be found after evaluating " filename)))

(defn path-error
  [err f args-type]
  (let [err (string/format
             ```
              ```

Encountered error generating path for site spec %s:
             


@@ 33,16 33,16 @@ Encountered error generating path for site spec %s:
Expected path function signature:
(defn %s %s "foo/bar/...")
             ```
             (spec-name)
             err
             (f-name f)
             (case args-type :d "[data]" :di "[data item]"))]
              (spec-name)
              err
              (f-name f)
              (case args-type :d "[data]" :di "[data item]"))]
    (error err)))

(defn renderer-error
  [err f args-type]
  (let [err (string/format
             ```
              ```

Encountered error rendering output for site spec %s:



@@ 51,17 51,17 @@ Encountered error rendering output for site spec %s:
Expected renderer function signature:
(defn %s %s "<html>...")
             ```
             (spec-name)
             err
             (format-renderer-context)
             (f-name f)
             (case args-type :d "[data]" :di "[data item]"))]
              (spec-name)
              err
              (format-renderer-context)
              (f-name f)
              (case args-type :d "[data]" :di "[data item]"))]
    (error err)))

(defn attrs-error
  [err attrs-f]
  (error (string/format
          ```
           ```

Encountered error getting attrs for data spec %s:



@@ 70,14 70,14 @@ Encountered error getting attrs for data spec %s:
Expected parse function signature:
(defn %s [src attrs] attrs)
          ```
          (spec-name)
          err
          (or (disasm attrs-f :name) "f"))))
           (spec-name)
           err
           (or (disasm attrs-f :name) "f"))))

(defn data-error
  [spec]
  (error (string/format
          ```
           ```

Received invalid data spec: %q



@@ 85,12 85,12 @@ Specification can be one of the following:
{:src (loader|path) :attrs parser)}
{:attrs attrs}
          ```
          spec)))
           spec)))

(defn site-error
  [spec]
  (error (string/format
          ```
           ```

Received invalid site spec: %q



@@ 100,4 100,4 @@ Specification can be one of the following:
{:dest (path|path-generator) :out (contents|renderer)} (write)
{:some site-selector :dest (path|path-generator)} (copy)
          ```
          spec)))
           spec)))

M src/generators.janet => src/generators.janet +7 -7
@@ 14,21 14,21 @@

(defn- apply-path [f args args-type]
  (try (maybe-apply f args)
       ([err fib] (error/path-error err f args-type))))
    ([err fib] (error/path-error err f args-type))))

(defn- apply-renderer [f args args-type]
  (try (maybe-apply f args)
       ([err fib]
        (error/renderer-error err f args-type))))
    ([err fib]
      (error/renderer-error err f args-type))))

(defn render-each-generator
  [bagatto env data spec-name filter site-selector path-generator renderer]
  (fn [parent]
    (set-cxt! bagatto env spec-name)
    

    (threads/print "Rendering " spec-name "...")
    (def res @[])
    

    (loop [item :in (data site-selector)]
      (if-let [_should-read (filter data item)
               path (apply-path path-generator [data item] :di)


@@ 58,7 58,7 @@

    (threads/print "Rendering " spec-name "...")
    (def res @[])
          

    (if-let [path (apply-path path-generator [data] :d)
             contents (apply-renderer renderer [data] :d)]
      (array/push res [:write path contents]))


@@ 71,7 71,7 @@

    (threads/print "Generating path for " spec-name "...")
    (def res @[])
    

    (if-let [item (data site-selector)
             from (item :path)
             to (apply-path path-generator [data] :d)]

M src/loaders.janet => src/loaders.janet +10 -10
@@ 2,11 2,11 @@
(import src/threads)

(defn- make-attrs [parser filename &opt file-contents]
    (let [base-attrs @{:path filename :contents file-contents}]
      (if file-contents
        (try (parser file-contents base-attrs) 
             ([err fib] (error/attrs-error err parser)))    
        base-attrs)))
  (let [base-attrs @{:path filename :contents file-contents}]
    (if file-contents
      (try (parser file-contents base-attrs)
        ([err fib] (error/attrs-error err parser)))
      base-attrs)))

(defn from-file-spec-loader
  ```


@@ 21,12 21,12 @@
                {:each ind}
                (do
                  (threads/print "[" spec-name "] Loading " (length ind) " files")
                  (->  (seq [spec :in ind]
                           (if (indexed? spec)
                             (make-attrs parser ;spec)
                             (make-attrs parser spec)))
                  (-> (seq [spec :in ind]
                        (if (indexed? spec)
                          (make-attrs parser ;spec)
                          (make-attrs parser spec)))
                      (transform-f)))
                                      

                {:some spec}
                (make-attrs parser ;spec))]
      (:send parent [:res spec-name res]))))

M src/multimarkdown.janet => src/multimarkdown.janet +6 -6
@@ 2,14 2,14 @@

(defn- metadata-keys
  [md]
  (->> (sh/$< echo ,(string md) | multimarkdown -m --)
  (->> (sh/$< echo ,(string md) |multimarkdown -m --)
       (string/trim)
       (string/split "\n")
       (filter (complement empty?))))

(defn- metadata-value
  [md key]
  (->> (sh/$< echo ,(string md) | multimarkdown -e ,key --)
  (->> (sh/$< echo ,(string md) |multimarkdown -e ,key --)
       (string/trim)))

(defn metadata [md]


@@ 21,7 21,7 @@
  (default smart true)

  (->>
   (if smart
     (sh/$< echo ,(string md) | multimarkdown -s --)
     (sh/$< echo ,(string md) | multimarkdown --nosmart -s --))
   (string/trim)))
    (if smart
      (sh/$< echo ,(string md) |multimarkdown -s --)
      (sh/$< echo ,(string md) |multimarkdown --nosmart -s --))
    (string/trim)))

M src/threads.janet => src/threads.janet +6 -8
@@ 44,22 44,22 @@
  is empty, the main thread sends a kill message to all writers.
  ```
  [specs init handler &opt pool-size]
  

  (def pool @{})
  (default pool-size default-pool-size)

  (print "Starting worker pool with " pool-size " workers...")
  

  (loop [t-id :range [0 pool-size]]
    (let [f (demand-worker t-id init handler)
          t (thread/new f worker-mailbox)]
      (put pool t-id t)))
  

  (var q (array/slice specs))
  (while (not (zero? (length q)))
    (let [t-id (thread/receive timeout)
          spec (array/pop q)
          dispatch [:consume spec] ]
          dispatch [:consume spec]]
      (:send (pool t-id) dispatch)))

  (each t pool (:send t :die))


@@ 82,11 82,11 @@
  [jobs]

  (print "Beginning " (length jobs) " jobs...")
  

  (let [workers (seq [job :in jobs] (one-time-worker job))
        res @{}]
    (var got-back 0)
  

    (while (< got-back (length jobs))
      (match (thread/receive timeout)
        [:res k v]


@@ 96,5 96,3 @@

    (print "Finished jobs.")
    res))



M src/writers.janet => src/writers.janet +5 -5
@@ 15,22 15,22 @@
        (unless (in path-cache s)
          (util/mkpath s path-cache)
          (put path-cache s true))))
    

    (match msg
      :die (thread/exit)
      

      [:consume [:write path contents]]
      (let [path (if output-dir (path/join output-dir path) path)
            ppath (path/dirname path)]
        

        (threads/print "[WRITE] " path)
        (ensure-path ppath)
        (spit path contents))
      

      [:consume [:copy from to]]
      (let [to (if output-dir (path/join output-dir to) to)
            ppath (path/dirname to)]
        

        (threads/print "[COPY] " to)
        (ensure-path ppath)
        (util/copy-file from to path-cache)))))