~drewr/directory

6c7fd3e930010130bd9445e1c3bb18a5f0c634cd — Drew Raines 1 year, 7 months ago 832f4ae
Separate external systems from global state
M env/dev/clj/user.clj => env/dev/clj/user.clj +7 -4
@@ 1,7 1,7 @@
(ns user
  "Userspace functions you can run by default in your local REPL."
  (:require
   [directory.config :refer [env]]
   [directory.config :refer [config]]
   [clojure.pprint]
   [clojure.spec.alpha :as s]
   [expound.alpha :as expound]


@@ 18,9 18,12 @@
  []
  (mount/start-without #'directory.core/repl-server))

(defn start-with-es [url]
  (mount/start-with-args {:es {:url url}}
                         #'directory.config/env
(defn start-custom [url bucket]
  (mount/start-with-args {:es {:url url}
                          :bucket bucket}
                         #'directory.config/config
                         #'directory.elasticsearch/elasticsearch
                         #'directory.gcloud/gcloud
                         #'directory.handler/init-app
                         #'directory.handler/app-routes
                         #'directory.core/http-server))

M env/dev/resources/config.edn => env/dev/resources/config.edn +2 -3
@@ 1,6 1,5 @@
{:options
 {:port 3000
  :url "http://localhost:3000"}
{:port 3000
 :url "http://localhost:3000"
 :es
 {:url "http://localhost:9200"
  :index "directory-dev"}

M env/prod/resources/config.edn => env/prod/resources/config.edn +2 -2
@@ 1,4 1,4 @@
{:prod true
 :options {:port 3000
           :url "https://c2c.draines.com"}
 :port 3000
 :url "https://c2c.draines.com"
 :es {:index "directory-prod"}}

M env/test/resources/config.edn => env/test/resources/config.edn +1 -2
@@ 1,5 1,4 @@
{:options
 {:port 3001}
{:port 3001
 :es
 {:url "http://localhost:9200"
  :index "directory-test"}}

M src/directory/config.clj => src/directory/config.clj +5 -24
@@ 1,31 1,12 @@
(ns directory.config
  (:require [cprop.core :refer [load-config]]
            [cprop.source :as source]
            [directory.elasticsearch :as es]
            [directory.gcloud :as g]
            [elasticsearch.connection :as es-meta]
            [elasticsearch.connection.http :as es-http]
            [clojure.tools.logging :as log]
            [mount.core :refer [args defstate]]))

(defstate env
  :start
  (let [cfg (load-config
             :merge
             [(args)
              (source/from-system-props)
              (source/from-env)])
        conn (es-http/make {:url (get-in cfg [:es :url])})
        es-ver (-> conn es-meta/version :number)
        es-host (format "%s:%d"
                        (-> conn :settings :server-name)
                        (-> conn :settings :server-port))
        gcloud (g/default-storage-service)]
    (if es-ver
      (log/infof "connected to elasticsearch %s on %s" es-ver es-host)
      (throw
       (ex-info "elasticsearch connection doesn't work" {:host es-host})))
    (es/make-index conn (get-in cfg [:es :index]))
    (assoc cfg
      :conn conn
      :gcloud gcloud)))
(defstate config :start (load-config
                         :merge
                         [(args)
                          (source/from-system-props)
                          (source/from-env)]))

M src/directory/core.clj => src/directory/core.clj +6 -6
@@ 2,7 2,7 @@
  (:require [directory.handler :as handler]
            [directory.nrepl :as nrepl]
            [luminus.http-server :as http]
            [directory.config :refer [env]]
            [directory.config :refer [config]]
            [clojure.tools.cli :refer [parse-opts]]
            [clojure.tools.logging :as log]
            [mount.core :as mount])


@@ 23,19 23,19 @@
(mount/defstate ^{:on-reload :noop} http-server
  :start
  (http/start
   (-> env
   (-> config
       (update :io-threads #(or % (* 2 (.availableProcessors (Runtime/getRuntime)))))
       (assoc  :handler (handler/app))
       (update :port #(or (-> env :options :port) %))
       (update :port #(or (config :port) %))
       (select-keys [:handler :host :port])))
  :stop
  (http/stop http-server))

(mount/defstate ^{:on-reload :noop} repl-server
  :start
  (when (env :nrepl-port)
    (nrepl/start {:bind (env :nrepl-bind)
                  :port (env :nrepl-port)}))
  (when (config :nrepl-port)
    (nrepl/start {:bind (config :nrepl-bind)
                  :port (config :nrepl-port)}))
  :stop
  (when repl-server
    (nrepl/stop repl-server)))

M src/directory/elasticsearch.clj => src/directory/elasticsearch.clj +19 -1
@@ 5,8 5,10 @@
            [clojure.spec.alpha :as s]
            [clojure.string :as str]
            [clojure.tools.logging :as log]
            [directory.config :refer [config]]
            [directory.model.business :as biz]
            [elasticsearch.async :as async]
            [elasticsearch.connection :as es-meta]
            [elasticsearch.connection.http :as conn]
            [elasticsearch.common :refer [format-uri request]]
            [elasticsearch.document :as doc]


@@ 14,11 16,28 @@
            [elasticsearch.scroll :as scroll]
            [java-time.api :as time]
            [jsonista.core :as json]
            [mount.core :refer [defstate]]
            [robert.bruce :refer [try-try-again *try*] :as try]
            [trptcolin.versioneer.core :as version]
            [slingshot.slingshot :refer [try+]])
  (:import (org.tukaani.xz XZOutputStream LZMA2Options)))

(declare make-index)

(defstate elasticsearch
  :start
  (let [conn (conn/make {:url (get-in config [:es :url])})
        es-ver (-> conn es-meta/version :number)
        es-host (format "%s:%d"
                        (-> conn :settings :server-name)
                        (-> conn :settings :server-port))]
    (if es-ver
      (log/infof "connected to elasticsearch %s on %s" es-ver es-host)
      (throw
       (ex-info "elasticsearch connection doesn't work" {:host es-host})))
    (make-index conn (get-in config [:es :index]))
    conn))

;; passthrough aliases to simplify imports in other namespaces
(def get doc/get)
(def index doc/index)


@@ 236,4 255,3 @@
(defn gzip-output-stream [channel]
  (java.util.zip.GZIPOutputStream.
   (java.nio.channels.Channels/newOutputStream channel)))


M src/directory/gcloud.clj => src/directory/gcloud.clj +4 -1
@@ 1,5 1,6 @@
(ns directory.gcloud
  (:require [clojure.java.io :as io])
  (:require [clojure.java.io :as io]
            [mount.core :refer [defstate]])
  (:import (java.nio.charset StandardCharsets)
           (java.nio.channels FileChannel)
           (java.nio.file Path Paths StandardOpenOption)


@@ 16,6 17,8 @@
  (-> (StorageOptions/getDefaultInstance)
      .getService))

(defstate gcloud :start (default-storage-service))

(defn make-blob-id
  ([bucket name]
   (BlobId/of bucket name))

M src/directory/layout.clj => src/directory/layout.clj +2 -2
@@ 2,7 2,7 @@
  (:require
   [clojure.java.io]
   [clojure.tools.logging :as log]
   [directory.config :refer [env]]
   [directory.config :refer [config]]
   [selmer.parser :as parser]
   [selmer.filters :as filters]
   [markdown.core :refer [md-to-html-string]]


@@ 37,7 37,7 @@
    {:status 200
     :body body
     :headers {"Content-Type" "text/html; charset=utf-8"
               "Cache-Control" (or (env :home-cache-control) "s-maxage=60")
               "Cache-Control" (or (config :home-cache-control) "s-maxage=60")
               "Content-Length" (-> body .getBytes count str)}}))

(defn login-page

M src/directory/main.clj => src/directory/main.clj +3 -3
@@ 1,13 1,13 @@
(ns directory.main
  (:gen-class)
  (:require [directory.config :refer [env]]
  (:require [directory.config :refer [config]]
            [directory.elasticsearch :as es]
            [directory.model.business :as biz]
            [clojure.tools.logging :as log]
            [mount.core :as mount]))

(defn load-biz [location]
  (mount/start #'directory.config/env)
  (mount/start #'directory.config/config)
  (log/infof "loading businesses from %s" location)
  (es/import-businesses (env :conn) (-> env :es :index) location)
  (es/import-businesses es/elasticsearch (-> config :es :index) location)
  (shutdown-agents))

M src/directory/middleware.clj => src/directory/middleware.clj +1 -1
@@ 10,7 10,7 @@
   [ring.middleware.anti-forgery :refer [wrap-anti-forgery]]
   [directory.middleware.formats :as formats]
   [muuntaja.middleware :refer [wrap-format wrap-params]]
   [directory.config :refer [env]]
   [directory.config :refer [config]]
   [ring.middleware.flash :refer [wrap-flash]]
   [ring.adapter.undertow.middleware.session :refer [wrap-session]]
   [ring.middleware.defaults :refer [site-defaults wrap-defaults]]

M src/directory/routes/home.clj => src/directory/routes/home.clj +22 -22
@@ 1,6 1,6 @@
(ns directory.routes.home
  (:require
   [directory.config :refer [env]]
   [directory.config :refer [config]]
   [directory.elasticsearch :as es]
   [directory.gcloud :as g]
   [directory.layout :as layout]


@@ 31,7 31,7 @@
(defn fully-qualify
  "Prepend uri with scheme and hostname"
  [uri]
  (let [base (-> env :options :url)]
  (let [base (config :url)]
    (format "%s%s" (or base "") uri)))

(defn md5


@@ 45,8 45,8 @@
    ""))

(defn last-modified []
  (let [conn (-> env :conn)
        index (-> env :es :index)
  (let [conn es/elasticsearch
        index (-> config :es :index)
        q {:query {:match_all {}}
           :sort {:updated-at :desc}
           :size 1


@@ 92,7 92,7 @@
      (layout/render-cached
       request "business-listing.html"
       {:businesses
        (->> (es/search (env :conn) (-> env :es :index)
        (->> (es/search es/elasticsearch (-> config :es :index)
                        {:body
                         {:query {:match {:business.status "active"}}
                          :_source [:updated-at


@@ 128,7 128,7 @@
(defn admin-home [request]
  (layout/render request "admin/home.html"
                 {:businesses
                  (->> (es/search (env :conn) (-> env :es :index)
                  (->> (es/search es/elasticsearch (-> config :es :index)
                                  {:body {:query {:match_all {}}
                                          :sort :business.name.keyword
                                          :size 1000}})


@@ 150,12 150,12 @@
(defn business-submit [request]
  (let [doc (process-business (-> request :params))
        created-id (:_id
                    (es/index (env :conn) (-> env :es :index)
                    (es/index es/elasticsearch (-> config :es :index)
                              {:body
                               (assoc doc
                                 :updated-at
                                 (System/currentTimeMillis))}))
        name (-> (es/get (env :conn) (-> env :es :index) created-id)
        name (-> (es/get es/elasticsearch (-> config :es :index) created-id)
                 :_source :name)]
    (layout/render request "business-form.html" {:id created-id
                                                 :notification {:type "success"


@@ 165,7 165,7 @@
(def dbg (atom nil))

(defn get-industries []
  (->> (es/search (env :conn) (-> env :es :index)
  (->> (es/search es/elasticsearch (-> config :es :index)
                  {:body
                   {:size 0
                    :aggs {:industries


@@ 186,9 186,9 @@
               :business (dissoc biz :id))
        es-resp (if new?
                  (es/index
                   (env :conn) (-> env :es :index) {:body body})
                   es/elasticsearch (-> config :es :index) {:body body})
                  (es/index
                   (env :conn) (-> env :es :index) (:id biz) {:body body}))]
                   es/elasticsearch (-> config :es :index) (:id biz) {:body body}))]
    (assoc-in (redirect (fully-qualify "/admin"))
      [:flash (if new? :created :saved)] (dissoc biz :avatar))))



@@ 200,8 200,8 @@
                  :industries (get-industries)}))

(defn get-business [request]
  (let [doc (es/get (env :conn)
                    (-> env :es :index)
  (let [doc (es/get es/elasticsearch
                    (-> config :es :index)
                    (-> request :path-params :id))
        biz (-> (-> doc :_source :business)
                (assoc :id (:_id doc)))]


@@ 216,7 216,7 @@
  (log/debug (pr-str 'get-image (:path-params request)))
  (let [id (-> request :path-params :business-id)
        doc (try+
             (es/get (env :conn) (-> env :es :index) id)
             (es/get es/elasticsearch (-> config :es :index) id)
             (catch [:status 404] {}))
        avatar (-> doc :_source :business :avatar)
        bytes (when (:image avatar)


@@ 228,7 228,7 @@
                 ;; We can cache aggressively because the avatar URL
                 ;; is immutable and only updated when the image
                 ;; changes.
                 "Cache-Control" (or (env :image-cache-control)
                 "Cache-Control" (or (config :image-cache-control)
                                     "public, max-age=604800, immutable")}
       :body bytes}
      {:status 404


@@ 237,31 237,31 @@

(defn backup-database [request]
  (let [index (or (-> request :path-params :index)
                  (-> env :es :index))
                  (-> config :es :index))
        backup-filename (es/make-backup-file-basename
                         index ".backup.gz")
        blob (g/make-blob
              (env :bucket) backup-filename "text/plain; charset=UTF-8" "gzip")]
              (config :bucket) backup-filename "text/plain; charset=UTF-8" "gzip")]
    (log/debug (pr-str 'backup blob))
    {:status 200
     :headers {"Content-Type" "text/plain"}
     :body (str
            (with-open [out (es/gzip-output-stream
                             (g/make-blob-channel (env :gcloud) blob))]
              (es/dump-index-to-gzip (env :conn) index out))
                             (g/make-blob-channel g/gcloud blob))]
              (es/dump-index-to-gzip es/elasticsearch index out))
            " "
            "gs://" (.getBucket blob) "/" (.getName blob)
            "\n")}))

(defn backups [request]
  (let [index (or (-> request :path-params :index)
                  (-> env :es :index))]
                  (-> config :es :index))]
    (log/debug (pr-str 'list-backups))
    {:status 200
     :headers {"Content-Type" "text/plain"}
     :body
     (str
      (->> (g/list-objects (env :gcloud) (env :bucket) index)
      (->> (g/list-objects g/gcloud (config :bucket) index)
           (map #(format "%s %s %d %s %d"
                         (:created-at %)
                         (:name %)


@@ 273,7 273,7 @@

(defn login [request]
  (let [{:keys [username password]} (:params request)
        admin-user (env :admin)]
        admin-user (config :admin)]
    (if (= [username password] [(:username admin-user)
                                (:password admin-user)])
      (do