62022576c733d73b4a2bc1868f38db596e0c4088 — Kridsada Thanabulpong 11 days ago 10bc6ba
Cleanup Docker and docker-compose.
21 files changed, 666 insertions(+), 321 deletions(-)

M .builds/debian.yml
M .builds/freebsd.yml
M Dockerfile
M Makefile
R vendor/{honcho/Procfile.dev => e.dev}
M README.md
D Vagrantfile
A docker-compose.override.yml
A docker-compose.yml
A docker-gen.sh
M fanboi2/cmd/ctl.py
R vendor/rootfs/{entrypoint => int}
A scripts/bootstrap.sh
A scripts/initdb.sh
R vendor/builds/{utils.sh => sh} -rw-r--r-- => -rwxr-xr-x
M setup.py
D vendor/docker/README.rst
D vendor/docker/docker-compose.override.yml
D vendor/docker/docker-compose.yml
D vendor/vagrant/bootstrap_builder.sh
D vendor/vagrant/bootstrap_web.sh
M .builds/debian.yml => .builds/debian.yml +2 -2
@@ 16,7 16,7 @@ gs_bucket: gs://builds.fanboi.ch
  tasks:
    - setup: |
-       . fanboi2/vendor/builds/utils.sh
+       . fanboi2/scripts/utils.sh
        cd fanboi2/ || exit 1
  
        _local=$(git rev-parse HEAD)


@@ 39,7 39,7 @@ echo 1 | sudo tee /proc/sys/kernel/unprivileged_userns_clone
        gcloud auth activate-service-account --key-file=$HOME/.creds/fanboi-gcp-builds.json
    - build: |
-       . fanboi2/vendor/builds/utils.sh
+       . fanboi2/scripts/utils.sh
        cd fanboi2/ || exit 1
  
        restore_cache $gs_bucket $HOME img Dockerfile || true

M .builds/freebsd.yml => .builds/freebsd.yml +8 -4
@@ 26,7 26,7 @@ sudo npm install -g yarn
        gcloud auth activate-service-account --key-file=$HOME/.creds/fanboi-gcp-builds.json
    - build: |
-       . fanboi2/vendor/builds/utils.sh
+       . fanboi2/scripts/utils.sh
        cd fanboi2/ || exit 1
  
        restore_cache $gs_bucket $HOME pip-cache setup.py || true


@@ 34,19 34,23 @@ make prod
        make assets
    - test: |
-       . fanboi2/vendor/builds/utils.sh
+       . fanboi2/scripts/utils.sh
        cd fanboi2/ || exit 1
  
+       sudo service postgresql oneinitdb
        sudo sysrc postgresql_enable=YES
-       sudo service postgresql initdb
+       sudo sysrc postgresql_flags="-l /var/log/postgresql.log"
        (
          printf "local all all trust\\n"
          printf "host all all 127.0.0.1/32 trust\\n"
          printf "host all all ::1/128 trust\\n"
        ) | sudo tee /var/db/postgres/data10/pg_hba.conf
  
+       sudo touch /var/log/postgresql.log
+       sudo chown postgres /var/log/postgresql.log
+ 
        sudo service postgresql start
-       trap "sudo service postgresql stop" 0 1 2 3 6 9 14 15
+       trap "sudo service postgresql stop" 0 1 2 3 6 14 15
        sudo -u postgres createuser -ds fanboi2
        sudo -u postgres createdb -U fanboi2 fanboi2_test
  

M Dockerfile => Dockerfile +17 -22
@@ 26,9 26,11 @@ && rm s6-overlay.tar.gz \
   && apk del .s6-fetch
  
- ENV HOME /tmp
- ENV VENVDIR /venv
- ENV BUILDDIR /build
+ ENV HOME /data
+ 
+ ENV DATADIR /data
+ ENV VENVDIR /data/venv
+ ENV BUILDDIR /data/build
  
  WORKDIR /src
  


@@ 46,29 48,22 @@ postgresql-libs \
   && sed -i -e 's/^all:*/all: prod/' Makefile \
   && sed -i -e 's/^ASSETS_SRCS.*/ASSETS_SRCS ?=/' Makefile \
-  && make prod \
-  && rm -rf /tmp/.cache \
+  && addgroup -g 10000 fanboi2 \
+  && adduser -D -h /tmp -u 10000 -G fanboi2 fanboi2 \
+  && mkdir -p $DATADIR \
+  && chown -R "10000:10000" $DATADIR \
+  && chown -R "10000:10000" /src \
+  && s6-setuidgid fanboi2 make prod \
+  && rm -rf /data/.cache \
   && apk del .app-build
  
- COPY alembic.ini ./
- COPY fanboi2/ ./fanboi2
- COPY migration/ ./migration
- 
- COPY --from=assets /src/fanboi2/static ./fanboi2/static
- 
- COPY vendor/rootfs/ /
- 
- ARG user=fanboi2
- ARG group=fanboi2
- ARG uid=10000
- ARG gid=10000
+ COPY rootfs/ /
+ COPY --chown=fanboi2:fanboi2 alembic.ini ./
+ COPY --chown=fanboi2:fanboi2 fanboi2/ ./fanboi2
+ COPY --chown=fanboi2:fanboi2 migration/ ./migration
+ COPY --chown=fanboi2:fanboi2 --from=assets /src/fanboi2/static ./fanboi2/static
  
  RUN set -xe \
-  && addgroup -g ${gid} ${group} \
-  && adduser -D -h /tmp -u ${uid} -G ${group} ${user} \
-  && chown -R "${uid}:${gid}" /build \
-  && chown -R "${uid}:${gid}" /src \
-  && chown -R "${uid}:${gid}" /venv \
   && chmod +x /entrypoint
  
  ENTRYPOINT ["/init", "/entrypoint"]

M Makefile => Makefile +6 -8
@@ 2,8 2,8 @@ CFLAGS      += -I/usr/local/include
  HOSTNAME    != hostname -s
  
- BUILDDIR    ?= .$(HOSTNAME).build
- VENVDIR     ?= .$(HOSTNAME).venv
+ BUILDDIR    ?= builds/$(HOSTNAME)
+ VENVDIR     ?= $(BUILDDIR)/venv
  ENVFILE     ?= .env
  YARN        ?= yarn
  


@@ 19,7 19,7 @@ BUILDENV     = env LANG=en_US.UTF-8 LDFLAGS="$(LDFLAGS)" CFLAGS="$(CFLAGS)"
  RUNENV       = env LANG=en_US.UTF-8 $$(test -f $(ENVFILE) && cat $(ENVFILE))
  
- ASSETS_SRCS != find assets/ -type f
+ ASSETS_SRCS != [ -d assets ] && find assets/ -type f
  
  
  all: assets prod


@@ 103,10 103,8 @@ dev: $(BUILDDIR)/.build-dev
  
  
- devrun: dev $(BUILDDIR)/.build-assets
- 	$(HONCHO) start \
- 		-e $(ENVFILE) \
- 		-f vendor/honcho/Procfile.dev
+ devrun: dev assets
+ 	$(HONCHO) start -e $(ENVFILE) -f Procfile.dev
  
  
  devhook: dev


@@ 117,7 115,7 @@ $(RUNENV) $(FBCTL) serve --reload --workers=1 --threads=4
  
  
- devassets: $(BUILDDIR)/.build-assets
+ devassets: assets
  	$(YARN) run gulp watch
  
  

R vendor/honcho/Procfile.dev => Procfile.dev +0 -0

        
M README.md => README.md +27 -26
@@ 6,7 6,7 @@   ## Installation
  
- Fanboi2 has the following runtime requirements:
+ For production environment, Fanboi2 has the following runtime requirements:
  
  -   [Python 3.6](https://www.python.org/downloads/) with [Virtualenv](https://virtualenv.pypa.io/en/stable/)
  -   [PostgreSQL 10](https://www.postgresql.org/)


@@ 17,13 17,13 @@ -   [Node 8](https://nodejs.org/) (Node 10 will NOT work)
  -   [Yarn](https://yarnpkg.com/)
  
- After all packages are installed, you may now setup the application:
+ After all packages are installed, setup the application with:
  
      $ git clone https://git.sr.ht/~sirn/fanboi2 fanboi2
      $ cd fanboi2/
      $ make all -j2
  
- Then configure `.env` according to the configuring section below, then run:
+ Then configure `.env` according to the configuring section below, and run:
  
      $ make migrate
      $ make serve


@@ 47,38 47,39 @@ -   `SERVER_DEV` -- Boolean flag whether to enable dev console, default False
  -   `SERVER_SECURE` -- Boolean flag whether to only authenticate via HTTPS, default False.
  
- ## Contributing
+ ## Development
  
- Fanboi2 is open to any contributors, whether you are learning Python or an expert. To contribute to Fanboi2, it is highly recommended to use [Vagrant](https://www.vagrantup.com/) as it is currently replicating the production environment of [Fanboi Channel](https://fanboi.ch/) and perform all the necessary setup steps for you. Alternatively, if containers are your thing, you can find experimental, unsupported Docker Compose scripts in `vendor/docker/`.
- 
- ### Vagrant
- 
- 1.  Install [Vagrant](https://www.vagrantup.com/) of your preferred platform.
- 2.  Install [VirtualBox](https://www.virtualbox.org/) or other providers supported by Vagrant.
- 3.  Run vagrant up and read Getting Started while waiting.
- 4.  Run vagrant ssh to SSH into the development machine (remember to `cd /vagrant`).
- 
- In case you do not want to use Vagrant, you can install the dependencies from the installation section and run:
+ To setup Fanboi2 in development mode, run the following commands after performing production setup steps:
  
      $ make dev
      $ make devhook
  
- You can then configure the application (see configuration section) and run the server:
+ And run the server with (which will run everything required for development):
  
-     $ make migrate
      $ make devrun
  
- ### Docker
+ ### FreeBSD
+ 
+ [Fanboi Channel](https://fanboi.ch/) uses FreeBSD as its deploy target. We no longer provides Vagrantfile due to complexity to maintain the environment, however in case a FreeBSD VM is used (e.g. via Bhyve, Xhyve, Virtualbox or other virtual machine applications) we provide a `scripts/bootstrap.sh` script which should setup the environment to be as close to the production setup as much as possible. To use the script:
+ 
+ 1. Create a virtual machine running [FreeBSD 12.0-RELEASE](https://www.freebsd.org/releases/12.0R/relnotes.html)
+ 2. Setup your user with `sudo` and SSH.
+ 3. Run `cat builds/bootstrap.sh | ssh user@host sudo sh`
+ 4. Configure NFS mount or clone the project according to the instruction above.
+ 
+ ### Docker Compose
+ 
+ To ease the development, we also provide a [Docker Compose](https://docs.docker.com/compose/) file suitable for both development and evaluation purpose. Please note that the resulting Dockerfile is not being used in [Fanboi Channel](https://fanboi.ch/) and may regress from time to time. To use this `docker-compose.yml` for evaluation purpose, simply use the auto-configuration tool:
+ 
+     $ ./docker-gen.sh -o docker-compose.override.yml
+     $ docker-compose up -d
+ 
+ In case you wish to develop using Docker Compose, instead run:
  
- 1.  Install `Docker` and `Docker Compose`
- 2.  Copy Compose configuration files to the application's parent directory (`cp vendor/docker/docker-compose.* ../`)
- 3.  Modify the content of `docker-compose.yml`
-     1.  Generate both `AUTH_SECRET` and `SESSION_SECRET` tokens with `openssl rand -hex 32`
-     2.  Set a sensible PostgreSQL password
-     3.  This config assumes fanboi2 was cloned to `fanboi2`; update the build and mount paths if untrue
- 4.  Start the contraption with `docker-compose up` from the same directory as the config files.
+     $ ./docker-gen.sh -d -o docker-compose.override.yml
+     $ docker-compose up --build -d
  
- Images are published to [sirn/fanboi2](https://hub.docker.com/r/sirn/fanboi2/) on Docker Hub for every commit in master. By default, using Fanboi2 with the default `docker-compose.yml` will start server in development mode which aids debugging. To disable development server capabilities, remove or rename the file `docker-compose.override.yml`.
+ Please inspect and adjust `docker-compose.override.yml` as needed.
  
  ### Submitting changes
  


@@ 171,7 172,7 @@   ## License
  
- Copyright © 2013-2018, Kridsada Thanabulpong. All rights reserved.
+ Copyright © 2013-2019, Kridsada Thanabulpong. All rights reserved.
  
  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
  

D Vagrantfile => Vagrantfile +0 -23
@@ 1,23 0,0 @@-# -*- mode: ruby -*-
- # vi: set ft=ruby :
- 
- Vagrant.configure("2") do |config|
-   config.vm.define "web", primary: true do |web|
-     web.vm.box = "generic/freebsd11"
-     web.vm.network "private_network", ip: "10.200.80.100"
-     web.vm.network "forwarded_port", guest: 6543, host: 6543
-     web.vm.synced_folder ".", "/vagrant", type: "nfs", mount_options: ["actimeo=2"]
-     web.ssh.shell = "sh"
-     web.vm.provision :shell, privileged: true, path: "vendor/vagrant/bootstrap_web.sh"
-   end
- 
-   # The builder vm is for building Docker image in case you're not a fan of Docker.
-   # By default it's not created when running `vagrant up`.
-   # To use it, explicitly run `vagrant up builder`.
-   config.vm.define "builder", autostart: false do |builder|
-     builder.vm.box = "generic/debian9"
-     builder.vm.network "private_network", ip: "10.200.80.101"
-     builder.vm.synced_folder ".", "/vagrant", type: "nfs", mount_options: ["actimeo=2"]
-     builder.vm.provision :shell, privileged: true, path: "vendor/vagrant/bootstrap_builder.sh"
-   end
- end

A docker-compose.override.yml => docker-compose.override.yml +59 -0
@@ 0,0 1,59 @@
+ ---
+ version: "3.4"
+ 
+ x-presets:
+   fanboi2: &fanboi2
+     environment:
+       AUTH_SECRET: 1c2e2a936a56382159bdceecc4526b1bbc01ceffb05b6ba7318d346e208d1243
+       SESSION_SECRET: 28f2c6cb686e4eae10db6fb05626591ba5972a16f1aa0daf83bd9f563aaafd06
+       DATABASE_URL: postgresql://fanboi2:H-DK6fcehRQpTMTh2_U9yiJuco8YtQl1@postgres:5432/fanboi2
+       POSTGRESQL_TEST_DATABASE: postgresql://fanboi2:H-DK6fcehRQpTMTh2_U9yiJuco8YtQl1@postgres:5432/fanboi2_test
+       SERVER_DEV: "true"
+     image: sirn/fanboi2:dev
+     volumes:
+       - ./Makefile:/src/Makefile
+       - ./alembic.ini:/src/alembic.ini
+       - ./fanboi2:/src/fanboi2
+       - ./migration:/src/migration
+       - ./setup.cfg:/src/setup.cfg
+       - ./setup.py:/src/setup.py
+ 
+ services:
+   postgres:
+     environment:
+       DB_PASSWORD: H-DK6fcehRQpTMTh2_U9yiJuco8YtQl1
+ 
+   web:
+     <<: *fanboi2
+     build: .
+     command: [make, devserve]
+ 
+   assets:
+     image: sirn/fanboi2-assets:dev
+     build:
+       context: .
+       target: assets
+     volumes:
+       - ./Makefile:/src/Makefile
+       - ./assets:/src/assets
+       - ./fanboi2/static:/src/fanboi2/static
+       - ./gulpfile.js:/src/gulpfile.js
+       - ./package.json:/src/package.json
+       - ./tsconfig.json:/src/tsconfig.json
+       - ./yarn.lock:/src/yarn.lock
+     command: [make, devassets]
+ 
+   worker:
+     <<: *fanboi2
+     depends_on:
+       - web
+ 
+   beat:
+     <<: *fanboi2
+     depends_on:
+       - web
+ 
+   migrate:
+     <<: *fanboi2
+     depends_on:
+       - web

A docker-compose.yml => docker-compose.yml +73 -0
@@ 0,0 1,73 @@
+ ---
+ version: "3.4"
+ 
+ x-presets:
+   fanboi2: &fanboi2
+     image: sirn/fanboi2:latest
+     restart: always
+     networks:
+       - fbnet
+     depends_on:
+       - postgres
+       - redis
+     environment:
+       AUTH_SECRET: CHANGE_ME
+       SESSION_SECRET: CHANGE_ME
+       DATABASE_URL: postgresql://postgres:passw0rd@postgres:5432/fanboi2
+       REDIS_URL: redis://redis/0
+       CELERY_BROKER_URL: redis://redis/1
+ 
+ services:
+   postgres:
+     image: postgres:10.5-alpine
+     restart: always
+     volumes:
+       - postgres-db:/var/lib/postgresql/data
+       - ./scripts/initdb.sh:/docker-entrypoint-initdb.d/initdb.sh
+     networks:
+       - fbnet
+     environment:
+       DB_PASSWORD: passw0rd
+     healthcheck:
+       test: [CMD-SHELL, pg_isready, -U, postgres]
+       interval: 30s
+       retries: 3
+ 
+   redis:
+     image: redis:5.0-alpine
+     restart: always
+     volumes:
+       - redis-db:/data
+     networks:
+       - fbnet
+     healthcheck:
+       test: [CMD-SHELL, redis-cli, ping]
+       interval: 30s
+       retries: 3
+ 
+   web:
+     <<: *fanboi2
+     command:
+       - serve
+     ports:
+       - "6543:6543"
+ 
+   worker:
+     <<: *fanboi2
+     command: [worker]
+ 
+   beat:
+     <<: *fanboi2
+     command: [beat]
+ 
+   migrate:
+     <<: *fanboi2
+     restart: on-failure
+     command: [migrate]
+ 
+ networks:
+   fbnet:
+ 
+ volumes:
+   postgres-db:
+   redis-db:

A docker-gen.sh => docker-gen.sh +190 -0
@@ 0,0 1,190 @@
+ #!/bin/sh
+ 
+ set -e
+ 
+ ## Arguments handling
+ ##
+ 
+ print_usage() {
+     printf "Usage: %s [CONFIG...] [OPTS...]\\n" "$0"
+     printf "\\n"
+     printf "OPTS:\\n"
+     printf "\\n"
+     printf "     -h                      Print this help.\\n"
+     printf "     -d                      Generates a development overrides.\\n"
+     printf "     -o FILENAME             Output to the given file.\\n"
+     printf "\\n"
+     printf "CONFIG:\\n"
+     printf "\\n"
+     printf "     -i IMAGE                Specify an alternative image name.\\n"
+     printf "     -n NETWORK              Specify an external network.\\n"
+     printf "     -p DB_PASSWORD          Specify a PostgreSQL password.\\n"
+     printf "     -s AUTH_SECRET          Specify an AUTH_SECRET variable.\\n"
+     printf "     -S SESSION_SECRET       Specify a SESSION_SECRET variable.\\n"
+     printf "     -f ENV                  Specify an ENV file to load ENV from.\\n"
+     printf "\\n"
+ }
+ 
+ OPTIND=1
+ 
+ DEV_MODE=0
+ OUTPUT_FILENAME=""
+ IMAGE_NAME=""
+ EXTERNAL_NETWORK=""
+ DB_PASSWORD=""
+ AUTH_SECRET=""
+ SESSION_SECRET=""
+ ENV_FILE=""
+ 
+ while getopts "hdo:i:n:p:s:S:f:" opt; do
+     case "$opt" in
+         d ) DEV_MODE=1;;
+         o ) OUTPUT_FILENAME=$OPTARG;;
+         i ) IMAGE_NAME=$OPTARG;;
+         n ) EXTERNAL_NETWORK=$OPTARG;;
+         p ) DB_PASSWORD=$OPTARG;;
+         s ) AUTH_SECRET=$OPTARG;;
+         S ) SESSION_SECRET=$OPTARG;;
+         f ) ENV_FILE=$OPTARG;;
+         h ) print_usage; exit 2;;
+         * ) print_usage; exit 1;;
+     esac
+ done
+ 
+ shift $((OPTIND-1))
+ if [ "${1:-}" = "--" ]; then
+     shift
+ fi
+ 
+ 
+ ## Normalizing
+ ##
+ 
+ [ -z "$SESSION_SECRET" ] && SESSION_SECRET=$(openssl rand -hex 32)
+ [ -z "$AUTH_SECRET" ]    && AUTH_SECRET=$(openssl rand -hex 32)
+ [ -z "$DB_PASSWORD" ]    && DB_PASSWORD=$(openssl rand -base64 24 | tr '+/' '-_')
+ [ -z "$IMAGE_NAME" ]     && IMAGE_NAME="sirn/fanboi2:latest"
+ 
+ 
+ ## Presets
+ ##
+ 
+ YML_PRESETS="$YML_PRESETS
+ x-presets:
+   fanboi2: &fanboi2
+     environment:
+       AUTH_SECRET: $AUTH_SECRET
+       SESSION_SECRET: $SESSION_SECRET
+       DATABASE_URL: postgresql://fanboi2:$DB_PASSWORD@postgres:5432/fanboi2\
+ " # EOF
+ 
+ if [ $DEV_MODE = 0 ]; then
+     YML_PRESETS="$YML_PRESETS
+     image: $IMAGE_NAME\
+ " # EOF
+ else
+     YML_PRESETS="$YML_PRESETS
+       POSTGRESQL_TEST_DATABASE: postgresql://fanboi2:$DB_PASSWORD@postgres:5432/fanboi2_test
+       SERVER_DEV: \"true\"
+     image: sirn/fanboi2:dev
+     volumes:
+       - ./Makefile:/src/Makefile
+       - ./alembic.ini:/src/alembic.ini
+       - ./fanboi2:/src/fanboi2
+       - ./migration:/src/migration
+       - ./setup.cfg:/src/setup.cfg
+       - ./setup.py:/src/setup.py\
+ " # EOF
+ fi
+ 
+ if [ -n "$ENV_FILE" ]; then
+     YML_PRESETS="$YML_PRESETS
+     env_file:
+       - $ENV_FILE\
+ " # EOF
+ fi
+ 
+ 
+ ## Services
+ ##
+ 
+ YML_SERVICES="$YML_SERVICES
+   postgres:
+     environment:
+       DB_PASSWORD: $DB_PASSWORD
+ 
+   web:
+     <<: *fanboi2\
+ " # EOF
+ 
+ if [ $DEV_MODE = 1 ]; then
+     YML_SERVICES="$YML_SERVICES
+     build: .
+     command: [make, devserve]
+ 
+   assets:
+     image: sirn/fanboi2-assets:dev
+     build:
+       context: .
+       target: assets
+     volumes:
+       - ./Makefile:/src/Makefile
+       - ./assets:/src/assets
+       - ./fanboi2/static:/src/fanboi2/static
+       - ./gulpfile.js:/src/gulpfile.js
+       - ./package.json:/src/package.json
+       - ./tsconfig.json:/src/tsconfig.json
+       - ./yarn.lock:/src/yarn.lock
+     command: [make, devassets]
+ \
+ " # EOF
+ fi
+ 
+ YML_SERVICES="$YML_SERVICES
+   worker:
+     <<: *fanboi2
+     depends_on:
+       - web
+ 
+   beat:
+     <<: *fanboi2
+     depends_on:
+       - web
+ 
+   migrate:
+     <<: *fanboi2
+     depends_on:
+       - web\
+ " # EOF
+ 
+ 
+ ## Network
+ ##
+ 
+ if [ -n "$EXTERNAL_NETWORK" ]; then
+     YML_NETWORKS="
+   fbnet:
+     external:
+       name: $EXTERNAL_NETWORK\
+ " # EOF
+ fi
+ 
+ 
+ ## Output
+ ##
+ 
+ print_yaml() {
+     printf -- "---\\nversion: \"3.4\"\\n"
+     [ -n "$YML_PRESETS" ]  && printf "%s\\n"             "$YML_PRESETS"
+     [ -n "$YML_SERVICES" ] && printf "\\nservices:%s\\n" "$YML_SERVICES"
+     [ -n "$YML_NETWORKS" ] && printf "\\nnetworks:%s\\n" "$YML_NETWORKS"
+ }
+ 
+ if [ -n "$OUTPUT_FILENAME" ]; then
+     TMPFILE=$(mktemp)
+     trap 'rm -f $TMPFILE' 0 1 2 3 6 14 15
+     print_yaml | tee "$TMPFILE" >/dev/null
+     cp "$TMPFILE" "$OUTPUT_FILENAME"
+ else
+     print_yaml
+ fi

M fanboi2/cmd/ctl.py => fanboi2/cmd/ctl.py +16 -16
@@ 24,7 24,7 @@ """
  
  
- def shell(args):
+ def run_shell(args):
      """Run the interactive shell for the application."""
      from ..wsgi import app, config
      import pyramid.scripting


@@ 36,7 36,7 @@ )
  
  
- def serve(args):
+ def run_serve(args):
      """Run the web server for the application."""
      from gunicorn.app.base import BaseApplication
      from ..wsgi import app as wsgi_app


@@ 68,7 68,7 @@ FbserveApplication(wsgi_app, options).run()
  
  
- def gensecret(args):
+ def run_gensecret(args):
      """Generates a NaCl secret."""
      from pyramid_nacl_session import generate_secret
  


@@ 81,21 81,21 @@ parser = argparse.ArgumentParser()
      subparsers = parser.add_subparsers(dest="subparser", title="subcommands")
  
-     pshell = subparsers.add_parser("shell")
-     pshell.set_defaults(func=shell)
+     shell = subparsers.add_parser("shell")
+     shell.set_defaults(func=run_shell)
  
      cpu_count = multiprocessing.cpu_count()
-     pserve = subparsers.add_parser("serve")
-     pserve.add_argument("--port", type=int, default=6543)
-     pserve.add_argument("--host", default="0.0.0.0")
-     pserve.add_argument("--workers", type=int, default=(cpu_count * 2) + 1)
-     pserve.add_argument("--threads", type=int, default=1)
-     pserve.add_argument("--max-requests", type=int, default=1000)
-     pserve.add_argument("--reload", action="store_true")
-     pserve.set_defaults(func=serve)
- 
-     gsecret = subparsers.add_parser("gensecret")
-     gsecret.set_defaults(func=gensecret)
+     serve = subparsers.add_parser("serve")
+     serve.add_argument("--port", type=int, default=6543)
+     serve.add_argument("--host", default="0.0.0.0")
+     serve.add_argument("--workers", type=int, default=(cpu_count * 2) + 1)
+     serve.add_argument("--threads", type=int, default=1)
+     serve.add_argument("--max-requests", type=int, default=1000)
+     serve.add_argument("--reload", action="store_true")
+     serve.set_defaults(func=run_serve)
+ 
+     gensecret = subparsers.add_parser("gensecret")
+     gensecret.set_defaults(func=run_gensecret)
  
      args = parser.parse_args()
      if args.subparser is None:

R vendor/rootfs/entrypoint => rootfs/entrypoint +2 -1
@@ 5,4 5,5 @@ ifelse { test $1 == "serve" } { make serve }
  ifelse { test $1 == "worker" } { make worker }
  ifelse { test $1 == "migrate" } { make migrate }
- exec $@> 
\ No newline at end of file
+ ifelse { test $1 == "beat" } { make beat }
+ exec $@

A scripts/bootstrap.sh => scripts/bootstrap.sh +247 -0
@@ 0,0 1,247 @@
+ #!/bin/sh
+ 
+ ## Utils
+ ##
+ 
+ echo_clear() {
+     printf "\\033[1A\\r\\033[K"
+ }
+ 
+ echo_newline() {
+     printf "\\n"
+ }
+ 
+ echo_ok() {
+     printf "=> \\033[1;37m%s\\033[0;0m\\n" "$1"
+ }
+ 
+ echo_wait() {
+     printf "=> \\033[0;33m%s\\033[0;0m\\n" "$1"
+ }
+ 
+ echo_error() {
+     printf "=> \\033[0;31m%s\\033[0;0m\\n" "$1"
+ }
+ 
+ echo_info() {
+     printf "   \\033[0;36m%s\\033[0;0m\\n" "$1"
+ }
+ 
+ 
+ ## Sanity check
+ ##
+ 
+ echo_wait "Checking system..."
+ 
+ if [ "$(uname)" != "FreeBSD" ]; then
+     echo_error "Development environment setup script only supports FreeBSD."
+     echo_info "For other operating systems, please see README.md."
+     exit 1
+ fi
+ 
+ if [ "$(id -u)" != "0" ] || [ -z "$SUDO_USER" ]; then
+     echo_error "Must be run with sudo."
+     echo_info "If this was run over SSH, the correct command should be:"
+     echo_info "cat builds/bsd-bootstrap.sh | ssh host sudo sh"
+     exit 1
+ fi
+ 
+ LOGIN_USER=$SUDO_USER
+ 
+ 
+ ## Ports
+ ##
+ 
+ if [ -f "/usr/ports/Makefile" ]; then
+     echo_ok "FreeBSD Ports already exists."
+ else
+     echo_wait "Development setup script requires FreeBSD Ports. Fetching..."
+     portsnap auto
+ fi
+ 
+ if hash synth 2>&1; then
+     echo_ok "Synth is already installed."
+ else
+     echo_wait "Synth is not installed. Installing..."
+     make -C /usr/ports/ports-mgmt/synth install clean
+ fi
+ 
+ 
+ ## Installation
+ ##
+ 
+ mkdir -p /usr/local/etc/synth/
+ echo_wait "Configuring Synth..."
+ 
+ if [ -f /usr/local/etc/synth/LiveSystem-make.conf ]; then
+     echo_info "Configuration for LiveSytem already exists at /usr/local/etc/synth."
+     echo_info "Moving LiveSystem-make.conf into LiveSystem-make.conf.bak"
+     mv /usr/local/etc/synth/LiveSystem-make.conf /usr/local/etc/synth/LiveSystem-make.conf.bak
+ fi
+ 
+ cat <<EOF > /usr/local/etc/synth/LiveSystem-make.conf
+ DEFAULT_VERSIONS+=ssl=libressl
+ 
+ # ftp/curl
+ ftp_curl_UNSET+=GSSAPI_NONE TLS_SRP
+ ftp_curl_SET+=GSSAPI_HEIMDAL
+ 
+ # security/p5-GSSAPI <- ftp/curl
+ security_p5-GSSAPI_UNSET+=GSSAPI_BASE
+ security_p5-GSSAPI_SET+=GSSAPI_HEIMDAL
+ 
+ # www/node8
+ www_node8_SET+=BUNDLED_SSL
+ 
+ # www/yarn
+ www_yarn_UNSET+=NODE
+ www_yarn_SET+=NODE8
+ EOF
+ 
+ INSTALL_LIST="$(mktemp)"
+ trap 'rm $INSTALL_LIST' 0 1 2 3 6 14 15
+ 
+ cat <<EOF > "$INSTALL_LIST"
+ databases/postgresql10-server
+ databases/py-sqlite3@py36
+ databases/redis
+ databases/sqlite3
+ devel/git-lite
+ devel/py-pip@py36
+ devel/py-virtualenv@py36
+ ftp/curl
+ graphics/GraphicsMagick
+ lang/python36
+ net/openntpd
+ security/ca_root_nss
+ www/node8
+ www/npm-node8
+ www/yarn
+ EOF
+ 
+ echo_wait "Installing development dependencies..."
+ echo_info "This may take a (really) long time."
+ 
+ # Redirecting STDIN otherwise Synth will cause the rest of
+ # shell script to broke for some reason...
+ synth install "$INSTALL_LIST" </dev/null
+ 
+ 
+ ## Enabling services
+ ##
+ 
+ if service openntpd onestatus >/dev/null; then
+     echo_ok "OpenNTPd is already enabled."
+ else
+     echo_wait "Enabling OpenNTPd..."
+     sysrc openntpd_enable=YES
+     service openntpd start
+ fi
+ 
+ if service postgresql onestatus >/dev/null; then
+     echo_ok "PostgreSQL is already enabled."
+ else
+     echo_wait "Enabling PostgreSQL..."
+     service postgresql oneinitdb
+ 
+     # Note: pg_ctl will write its output to controlling terminal which
+     # will cause shell process to never terminate. See also pg_ctl(1).
+     sysrc postgresql_enable=YES
+     sysrc postgresql_flags="-l /var/log/postgresql.log"
+ 
+     touch /var/log/postgresql.log
+     chown postgres:postgres /var/log/postgresql.log
+ 
+     cat <<EOF > /var/db/postgres/data10/pg_hba.conf
+ local all all trust
+ host all all 127.0.0.1/32 trust
+ host all all ::1/128 trust
+ EOF
+ 
+     service postgresql start
+     sudo -u postgres createuser -ds "$LOGIN_USER" || true
+     sudo -u postgres createuser -ds fanboi2 || true
+ fi
+ 
+ if service redis onestatus >/dev/null; then
+     echo_ok "Redis is already enabled."
+ else
+     echo_wait "Enabling Redis..."
+     sysrc redis_enable=YES
+     service redis start
+ fi
+ 
+ # nfsclient doesn't provide status so we're checking
+ # the lock daemon instead.
+ if service lockd onestatus >/dev/null; then
+     echo_ok "NFS is already enabled."
+ else
+     echo_wait "Enabling NFS..."
+     sysrc nfs_client_enable=YES
+     sysrc rpc_lockd_enable="YES"
+     sysrc rpc_statd_enable="YES"
+     service nfsclient start
+     service lockd start
+     service statd start
+ fi
+ 
+ 
+ ## User configurations
+ ##
+ 
+ echo_wait "Setting up user environment..."
+ echo_info "You must setup NFS mount and run \`make devserver\` on your own."
+ echo_info "However initial configuration will be provided at .local/fanboi2/env"
+ echo_info "with ENVFILE configured."
+ 
+ sudo -u "$LOGIN_USER" sh <<EOF
+ cd "\$HOME" || exit
+ rm -f \
+    .bashrc \
+    .cshrc \
+    .lesshst \
+    .login \
+    .login-e \
+    .login_conf \
+    .mail_aliases \
+    .mailrc \
+    .profile \
+    .profile-e \
+    .rhosts \
+    .rnd \
+    .shrc
+ 
+ cat <<EOPROFILE > "\$HOME/.profile"
+ EDITOR=vim; export EDITOR
+ PAGER=more; export PAGER
+ LANG=en_US.UTF-8; export LANG
+ 
+ # Fanboi2
+ FBVAR=\$HOME/.local/fanboi2; export FBVAR
+ VIRTUALENV=virtualenv-3.6; export VIRTUALENV
+ BUILDDIR=\\\$FBVAR/build; export BUILDDIR
+ VENVDIR=\\\$FBVAR/venv; export VENVDIR
+ ENVFILE=\\\$FBVAR/env; export ENVFILE
+ PATH=\\\$VENVDIR/bin:\\\$PATH; export PATH
+ EOPROFILE
+ 
+ . "\$HOME/.profile"
+ mkdir -p \$FBVAR
+ 
+ cat <<EOENV > "\$ENVFILE"
+ CELERY_BROKER_URL=redis://127.0.0.1:6379/1
+ DATABASE_URL=postgresql://fanboi2:@127.0.0.1:5432/fanboi2_dev
+ POSTGRESQL_TEST_DATABASE=postgresql://fanboi2:@127.0.0.1:5432/fanboi2_test
+ REDIS_URL=redis://127.0.0.1:6379/0
+ SERVER_DEV=true
+ SERVER_HOST=0.0.0.0
+ SERVER_PORT=6543
+ SESSION_SECRET=\$(openssl rand -hex 32)
+ AUTH_SECRET=\$(openssl rand -hex 32)
+ EOENV
+ EOF
+ 
+ (
+     psql -U fanboi2 template1 -c "CREATE DATABASE fanboi2_dev;" || true
+     psql -U fanboi2 template1 -c "CREATE DATABASE fanboi2_test;" || true
+ ) >/dev/null 2>&1

A scripts/initdb.sh => scripts/initdb.sh +16 -0
@@ 0,0 1,16 @@
+ #!/bin/sh
+ 
+ set -e
+ 
+ if [ -z "$DB_PASSWORD" ]; then
+     printf "DB_PASSWORD is not present, exiting.\\n"
+     exit 1
+ fi
+ 
+ psql -v ON_ERROR_STOP=1 --username="$POSTGRES_USER" --dbname="$POSTGRES_DB" <<EOF
+ CREATE USER fanboi2 WITH ENCRYPTED PASSWORD '$DB_PASSWORD';
+ CREATE DATABASE fanboi2 WITH OWNER fanboi2;
+ CREATE DATABASE fanboi2_test WITH OWNER fanboi2;
+ GRANT ALL PRIVILEGES ON DATABASE fanboi2 TO fanboi2;
+ GRANT ALL PRIVILEGES ON DATABASE fanboi2_test TO fanboi2;
+ EOF

R vendor/builds/utils.sh => scripts/utils.sh +0 -0

        
M setup.py => setup.py +3 -2
@@ 11,7 11,7 @@   setup(
      name="fanboi2",
-     version="2018.12",
+     version="2019.02",
      description="Board engine behind fanboi.ch",
      long_description=LONG_DESCRIPTION,
      long_description_content_type="text/markdown",


@@ 43,6 43,7 @@ "gunicorn",
          "hiredis >=0.2, <0.3",
          "isodate",
+         "kombu >= 4.3, <4.4",
          "lark-parser >=0.6, <0.7",
          "misaka",
          "passlib",


@@ 64,7 65,7 @@ zip_safe=False,
      test_suite="fanboi2.tests",
      extras_require={
-         "dev": ["honcho", "pre-commit"],
+         "dev": ["honcho", "hupper", "pre-commit"],
          "test": ["nose", "coverage", "rednose"],
          "deploy": ["fabric", "patchwork", "invocations", "colorama"],
      },

D vendor/docker/README.rst => vendor/docker/README.rst +0 -25
@@ 1,25 0,0 @@-============================
- Docker Compose Configuration
- ============================
- 
- These configuration are intended to help you launch Fanboi2 and all its dependencies on one machine, mainly for development/staging. For a real production deployment, you will want to use container orchestration tools such as Docker Swarm or Kubernetes, which are beyond the scope of this document.
- 
- Setting Up
- ----------
- 
- These configuration files assume the following directory structure::
- 
-   project_root/
-   ├── docker-compose.yml
-   ├── docker-compose.override.yml
-   └── fanboi2/
-       └── <REPOSITORY CONTENT>
- 
- 1. Create the above directory structure by copying both `yml` files to the same level as the repository directory
- 2. Edit the content of `docker-compose.yml` and initialize the variables with freshly generated secret tokens
- 3. Run `docker-compose build && docker-compose up -d` from the context of `project_root` (*not* the repository root)
- 
- Disabling Code Reload
- ---------------------
- 
- Removing or renaming `docker-compose.override.yml` to something else will prevent automatic reloading and rebuilding of code, similar to when in production. Don't forget to re-run `docker-compose up -d` when you make any changes.

D vendor/docker/docker-compose.override.yml => vendor/docker/docker-compose.override.yml +0 -15
@@ 1,15 0,0 @@-version: '2.3'
- services:
- 
-   web:
-     environment:
-       SERVER_DEV: "true"
-     command: devserve
- 
-   assets:
-     build:
-       context: ./fanboi2
-       target: assets
-     volumes:
-       - ./fanboi2:/src
-     command: make devassets

D vendor/docker/docker-compose.yml => vendor/docker/docker-compose.yml +0 -39
@@ 1,39 0,0 @@-version: '2.3'
- 
- x-app:
-   &app-defaults
-   build: ./fanboi2
-   volumes:
-     - ./fanboi2:/src
-   links:
-     - postgres
-     - redis
-   environment:
-     AUTH_SECRET: <REPLACE WITH SECRET>
-     SESSION_SECRET: <REPLACE WITH SECRET>
-     DATABASE_URL: postgres://postgres:<REPLACE WITH PASSWORD>@postgres/fanboi2
-     REDIS_URL: redis://redis/0
-     CELERY_BROKER_URL: redis://redis/1
- 
- services:
-   web:
-     <<: *app-defaults
-     ports:
-       - '6543:6543'
- 
-   worker:
-     <<: *app-defaults
-     command: worker
- 
-   migrations:
-     <<: *app-defaults
-     command: migrate
- 
-   postgres:
-     image: postgres:10.5-alpine
-     environment:
-       POSTGRES_PASSWORD: <REPLACE WITH PASSWORD>
-       POSTGRES_DB: fanboi2
- 
-   redis:
-     image: redis:5.0-alpine

D vendor/vagrant/bootstrap_builder.sh => vendor/vagrant/bootstrap_builder.sh +0 -47
@@ 1,47 0,0 @@-#!/bin/sh
- set -xe
- 
- apt-get update
- apt-get install -y \
-         apt-transport-https \
-         ca-certificates \
-         curl \
-         dirmngr \
-         gnupg2 \
-         software-properties-common
- 
- #
- # Docker
- #
- 
- curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
- add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
- apt-get update
- apt-get install -y docker-ce docker-compose
- 
- systemctl enable docker
- systemctl start docker
- usermod -aG docker vagrant
- 
- #
- # IMG
- #
- 
- apt-get update
- apt-get install -y libseccomp-dev uidmap
- echo 1 > /proc/sys/kernel/unprivileged_userns_clone
- 
- _img_sha256="6b7b660fa0a4c4ab10aa2c2d7d586afdbc70cb33644995b0ee0e7f77ddcc2565"
- _img_version="v0.5.4"
- curl -fSL "https://github.com/genuinetools/img/releases/download/$_img_version/img-linux-amd64" -o "/usr/local/bin/img" \
-     && echo "${_img_sha256}  /usr/local/bin/img" | sha256sum -c - \
-     && chmod a+x "/usr/local/bin/img"
- 
- #
- # Google Cloud SDK
- #
- 
- curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
- apt-add-repository "deb [arch=amd64] http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main"
- apt-get update
- apt-get install -y google-cloud-sdk

D vendor/vagrant/bootstrap_web.sh => vendor/vagrant/bootstrap_web.sh +0 -91
@@ 1,91 0,0 @@-#!/bin/sh
- set -xe
- 
- sysrc hostname=vagrant
- hostname vagrant
- 
- # generic/freebsd11 image ship with broken rc.conf, i.e. use `firstboot-growfs`
- # instead of `firstboot_growfs`, causing lots of warnings when running pkg.
- if awk '! /^firstboot/' < /etc/rc.conf > /etc/rc.conf.tmp; then
-     mv /etc/rc.conf.tmp /etc/rc.conf
- fi
- 
- pkg update -qf
- 
- PACKAGES="bzip2 ca_root_nss curl git-lite gmake ntp sqlite3 sudo"
- PACKAGES="$PACKAGES postgresql10-server redis"
- PACKAGES="$PACKAGES python36 py36-pip py36-sqlite3 py36-virtualenv"
- PACKAGES="$PACKAGES node8 npm-node8"
- 
- # shellcheck disable=SC2086
- pkg install -qy $PACKAGES
- npm install -g yarn
- 
- if ! service ntpd onestatus >/dev/null; then
-     sysrc ntpd_enable=YES
-     ntpd -qg
-     service ntpd start
- fi
- 
- if ! service postgresql onestatus >/dev/null; then
-     sysrc postgresql_enable=YES
- 
-     service postgresql initdb
-     cat <<EOF > /var/db/postgres/data10/pg_hba.conf
- local all all trust
- host all all 127.0.0.1/32 trust
- host all all ::1/128 trust
- EOF
- 
-     service postgresql start
-     sudo -u postgres createuser -ds vagrant || true
-     sudo -u postgres createuser -ds fanboi2 || true
- fi
- 
- if ! service redis onestatus >/dev/null; then
-     sysrc redis_enable=YES
-     service redis start
- fi
- 
- chsh -s /bin/sh vagrant
- sudo -u vagrant sh <<EOF
- cd "\$HOME" || exit
- rm -f \
-    .bashrc \
-    .cshrc \
-    .lesshst \
-    .login \
-    .login-e \
-    .login_conf \
-    .mail_aliases \
-    .mailrc \
-    .profile \
-    .profile-e \
-    .rhosts \
-    .rnd \
-    .shrc
- 
- cat <<EOPROFILE > "\$HOME/.profile"
- EDITOR=vim; export EDITOR
- PAGER=more; export PAGER
- LANG=en_US.UTF-8; export LANG
- EOPROFILE
- 
- cd /vagrant || exit
- 
- psql template1 -c "CREATE DATABASE fanboi2_dev;"
- psql template1 -c "CREATE DATABASE fanboi2_test;"
- 
- cat <<EOENV > "/vagrant/.env"
- CELERY_BROKER_URL=redis://127.0.0.1:6379/1
- DATABASE_URL=postgresql://vagrant:@127.0.0.1:5432/fanboi2_dev
- REDIS_URL=redis://127.0.0.1:6379/0
- SERVER_DEV=true
- SERVER_HOST=0.0.0.0
- SERVER_PORT=6543
- SESSION_SECRET=\$(openssl rand -hex 32)
- AUTH_SECRET=\$(openssl rand -hex 32)
- EOENV
- 
- VIRTUALENV=virtualenv-3.6 make -j$(sysctl -n hw.ncpu) assets dev
- EOF