~subsetpark/bagatto

ref: 22683bebdf00ea054405e41739fc0ef5fdeb7666 bagatto/src/core.janet -rw-r--r-- 5.3 KiB
22683beb — Zach Smith Demand pool management: wait for all threads to exit before main thread exits 10 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
(import path)

(import src/util)
(import src/error)
(import src/threads)
(import src/loaders)
(import src/generators)
(import src/writers)

(defn- struct->table [s]
  (->> (or s @{}) (kvs) (splice) (table)))

(defn- set-defaults [spec]
  (table/setproto (struct->table spec)
                  (struct->table (dyn :bagatto-defaults))))


(defn load-data
  ```
  First phase of main business logic. `data-spec` contains a
  specification of all the sources necessary to generate our `attrs`
  entries, which are the data structures to be used in generating the
  site.

  A data specification can be in one of four formats:
  
  1. A simple struct with only an :attrs field. The attrs generated
  will simply be the value at `:attrs`.
  2. A struct with a :src that's a string path to a file relative from
  the current directory, and an :attrs function that will be called on
  the contents of the file.
  3. A struct with a :src that's a 0-arity function that will return a
  [filename file-contents] tuple, and an :attrs function that will be
  called on the file-contents.
  4. A struct with a :src that returns a fiber which will yield some
  finite number of [filename file-contents] tuples, and an :attrs
  function that will be called on each file-contents.
  ```
  [data-spec]

  (let [data-pairs (pairs data-spec)
        jobs (seq [[spec-name spec] :in data-pairs]

               (threads/print "Reading " spec-name " data spec...")

               (setdyn :error-context {:spec-name spec-name})

               (let [with-defaults (set-defaults spec)
                     transform-f (or (spec :transform) identity)]
                 (match with-defaults
                   ({:src loader :attrs parser} (function? loader))
                   (loaders/from-file-spec-loader spec-name loader parser transform-f)

                   ({:src path :attrs parser} (string? path))
                   (loaders/from-path-loader spec-name path parser)

                   {:attrs attrs}
                   (loaders/bare-attr-loader spec-name attrs)

                   _ (error/data-error with-defaults))))]
    (threads/distribute jobs)))

(defn produce-writer-specs
  ```
  Second phase of main business logic. `site` contains a specification
  for generating a website and `data` is all the source data we have
  to do it with. Here we generate a new writer specification (a tuple
  of path and contents) for each file in the website.
  ```
  [site data bagatto env]
  (let [site-pairs (pairs site)
        jobs (seq [[spec-name spec] :in site-pairs]

               (threads/print "Reading " spec-name " site spec...")

               (let [with-defaults (set-defaults spec)
                     filter (spec :filter)]
                 (default filter (fn [_site _item] true))

                 (match with-defaults
                   {:each site-selector
                    :dest path-generator
                    :out renderer}
                   (generators/render-each-generator bagatto
                                                     env
                                                     data
                                                     spec-name
                                                     filter
                                                     site-selector
                                                     path-generator
                                                     renderer)

                   {:each site-selector
                    :dest path-generator}
                   (generators/copy-each-generator bagatto
                                                   env
                                                   data
                                                   spec-name
                                                   filter
                                                   site-selector
                                                   path-generator)

                   {:dest path-generator
                    :out renderer}
                   (generators/render-generator bagatto
                                                env
                                                data
                                                spec-name
                                                path-generator
                                                renderer)

                   {:some site-selector
                    :dest path-generator}
                   (generators/copy-some-generator bagatto
                                                   env
                                                   data
                                                   spec-name
                                                   site-selector
                                                   path-generator)

                   _ (error/site-error with-defaults))))]

    (let [segments (-> (threads/distribute jobs) (values))]
      (array/concat ;segments))))

(defn evaluate-writer-specs
  ```
  Third phase of business logic : given a list of writer specs, render
  them into new files.
  ```
  [env writer-specs]

  (def output-dir (env :bagatto-output-dir))

  (threads/demand-pipeline writer-specs
                           writers/writer-init
                           (writers/handle-writes output-dir)
                           (min threads/default-pool-size (length writer-specs))))