~sirn/fanboi2

62022576c733d73b4a2bc1868f38db596e0c4088 — Kridsada Thanabulpong 2 years ago 10bc6ba
Cleanup Docker and docker-compose.
21 files changed, 666 insertions(+), 321 deletions(-)

M .builds/debian.yml
M .builds/freebsd.yml
M Dockerfile
M Makefile
R vendor/honcho/Procfile.dev => Procfile.dev
M README.md
D Vagrantfile
A docker-compose.override.yml
A docker-compose.yml
A docker-gen.sh
M fanboi2/cmd/ctl.py
R vendor/rootfs/entrypoint => rootfs/entrypoint
A scripts/bootstrap.sh
A scripts/initdb.sh
R vendor/builds/utils.sh => scripts/utils.sh -rw-r--r-- => -rwxr-xr-x
M setup.py
D vendor/docker/README.rst
D vendor/docker/docker-compose.override.yml
D vendor/docker/docker-compose.yml
D vendor/vagrant/bootstrap_builder.sh
D vendor/vagrant/bootstrap_web.sh
M .builds/debian.yml => .builds/debian.yml +2 -2
@@ 16,7 16,7 @@ environment:
  gs_bucket: gs://builds.fanboi.ch
tasks:
  - setup: |
      . fanboi2/vendor/builds/utils.sh
      . fanboi2/scripts/utils.sh
      cd fanboi2/ || exit 1

      _local=$(git rev-parse HEAD)


@@ 39,7 39,7 @@ tasks:
      echo 1 | sudo tee /proc/sys/kernel/unprivileged_userns_clone
      gcloud auth activate-service-account --key-file=$HOME/.creds/fanboi-gcp-builds.json
  - build: |
      . fanboi2/vendor/builds/utils.sh
      . fanboi2/scripts/utils.sh
      cd fanboi2/ || exit 1

      restore_cache $gs_bucket $HOME img Dockerfile || true

M .builds/freebsd.yml => .builds/freebsd.yml +8 -4
@@ 26,7 26,7 @@ tasks:
      sudo npm install -g yarn
      gcloud auth activate-service-account --key-file=$HOME/.creds/fanboi-gcp-builds.json
  - build: |
      . fanboi2/vendor/builds/utils.sh
      . fanboi2/scripts/utils.sh
      cd fanboi2/ || exit 1

      restore_cache $gs_bucket $HOME pip-cache setup.py || true


@@ 34,19 34,23 @@ tasks:
      make prod
      make assets
  - test: |
      . fanboi2/vendor/builds/utils.sh
      . fanboi2/scripts/utils.sh
      cd fanboi2/ || exit 1

      sudo service postgresql oneinitdb
      sudo sysrc postgresql_enable=YES
      sudo service postgresql initdb
      sudo sysrc postgresql_flags="-l /var/log/postgresql.log"
      (
        printf "local all all trust\\n"
        printf "host all all 127.0.0.1/32 trust\\n"
        printf "host all all ::1/128 trust\\n"
      ) | sudo tee /var/db/postgres/data10/pg_hba.conf

      sudo touch /var/log/postgresql.log
      sudo chown postgres /var/log/postgresql.log

      sudo service postgresql start
      trap "sudo service postgresql stop" 0 1 2 3 6 9 14 15
      trap "sudo service postgresql stop" 0 1 2 3 6 14 15
      sudo -u postgres createuser -ds fanboi2
      sudo -u postgres createdb -U fanboi2 fanboi2_test


M Dockerfile => Dockerfile +17 -22
@@ 26,9 26,11 @@ RUN set -xe \
 && rm s6-overlay.tar.gz \
 && apk del .s6-fetch

ENV HOME /tmp
ENV VENVDIR /venv
ENV BUILDDIR /build
ENV HOME /data

ENV DATADIR /data
ENV VENVDIR /data/venv
ENV BUILDDIR /data/build

WORKDIR /src



@@ 46,29 48,22 @@ RUN set -xe \
        postgresql-libs \
 && sed -i -e 's/^all:*/all: prod/' Makefile \
 && sed -i -e 's/^ASSETS_SRCS.*/ASSETS_SRCS ?=/' Makefile \
 && make prod \
 && rm -rf /tmp/.cache \
 && addgroup -g 10000 fanboi2 \
 && adduser -D -h /tmp -u 10000 -G fanboi2 fanboi2 \
 && mkdir -p $DATADIR \
 && chown -R "10000:10000" $DATADIR \
 && chown -R "10000:10000" /src \
 && s6-setuidgid fanboi2 make prod \
 && rm -rf /data/.cache \
 && apk del .app-build

COPY alembic.ini ./
COPY fanboi2/ ./fanboi2
COPY migration/ ./migration

COPY --from=assets /src/fanboi2/static ./fanboi2/static

COPY vendor/rootfs/ /

ARG user=fanboi2
ARG group=fanboi2
ARG uid=10000
ARG gid=10000
COPY rootfs/ /
COPY --chown=fanboi2:fanboi2 alembic.ini ./
COPY --chown=fanboi2:fanboi2 fanboi2/ ./fanboi2
COPY --chown=fanboi2:fanboi2 migration/ ./migration
COPY --chown=fanboi2:fanboi2 --from=assets /src/fanboi2/static ./fanboi2/static

RUN set -xe \
 && addgroup -g ${gid} ${group} \
 && adduser -D -h /tmp -u ${uid} -G ${group} ${user} \
 && chown -R "${uid}:${gid}" /build \
 && chown -R "${uid}:${gid}" /src \
 && chown -R "${uid}:${gid}" /venv \
 && chmod +x /entrypoint

ENTRYPOINT ["/init", "/entrypoint"]

M Makefile => Makefile +6 -8
@@ 2,8 2,8 @@ LDFLAGS     += -L/usr/local/lib
CFLAGS      += -I/usr/local/include
HOSTNAME    != hostname -s

BUILDDIR    ?= .$(HOSTNAME).build
VENVDIR     ?= .$(HOSTNAME).venv
BUILDDIR    ?= builds/$(HOSTNAME)
VENVDIR     ?= $(BUILDDIR)/venv
ENVFILE     ?= .env
YARN        ?= yarn



@@ 19,7 19,7 @@ PYTHON       = $(VENVDIR)/bin/python3
BUILDENV     = env LANG=en_US.UTF-8 LDFLAGS="$(LDFLAGS)" CFLAGS="$(CFLAGS)"
RUNENV       = env LANG=en_US.UTF-8 $$(test -f $(ENVFILE) && cat $(ENVFILE))

ASSETS_SRCS != find assets/ -type f
ASSETS_SRCS != [ -d assets ] && find assets/ -type f


all: assets prod


@@ 103,10 103,8 @@ assets: $(BUILDDIR)/.build-assets
dev: $(BUILDDIR)/.build-dev


devrun: dev $(BUILDDIR)/.build-assets
	$(HONCHO) start \
		-e $(ENVFILE) \
		-f vendor/honcho/Procfile.dev
devrun: dev assets
	$(HONCHO) start -e $(ENVFILE) -f Procfile.dev


devhook: dev


@@ 117,7 115,7 @@ devserve: dev
	$(RUNENV) $(FBCTL) serve --reload --workers=1 --threads=4


devassets: $(BUILDDIR)/.build-assets
devassets: assets
	$(YARN) run gulp watch



R vendor/honcho/Procfile.dev => Procfile.dev +0 -0
M README.md => README.md +27 -26
@@ 6,7 6,7 @@ Board engine behind [Fanboi Channel](https://fanboi.ch/) written in Python.

## Installation

Fanboi2 has the following runtime requirements:
For production environment, Fanboi2 has the following runtime requirements:

-   [Python 3.6](https://www.python.org/downloads/) with [Virtualenv](https://virtualenv.pypa.io/en/stable/)
-   [PostgreSQL 10](https://www.postgresql.org/)


@@ 17,13 17,13 @@ Additionally, the following packages are build-time requirements for compiling a
-   [Node 8](https://nodejs.org/) (Node 10 will NOT work)
-   [Yarn](https://yarnpkg.com/)

After all packages are installed, you may now setup the application:
After all packages are installed, setup the application with:

    $ git clone https://git.sr.ht/~sirn/fanboi2 fanboi2
    $ cd fanboi2/
    $ make all -j2

Then configure `.env` according to the configuring section below, then run:
Then configure `.env` according to the configuring section below, and run:

    $ make migrate
    $ make serve


@@ 47,38 47,39 @@ Fanboi2 uses environment variable to configure the application. In case `make` i
-   `SERVER_DEV` -- Boolean flag whether to enable dev console, default False
-   `SERVER_SECURE` -- Boolean flag whether to only authenticate via HTTPS, default False.

## Contributing
## Development

Fanboi2 is open to any contributors, whether you are learning Python or an expert. To contribute to Fanboi2, it is highly recommended to use [Vagrant](https://www.vagrantup.com/) as it is currently replicating the production environment of [Fanboi Channel](https://fanboi.ch/) and perform all the necessary setup steps for you. Alternatively, if containers are your thing, you can find experimental, unsupported Docker Compose scripts in `vendor/docker/`.

### Vagrant

1.  Install [Vagrant](https://www.vagrantup.com/) of your preferred platform.
2.  Install [VirtualBox](https://www.virtualbox.org/) or other providers supported by Vagrant.
3.  Run vagrant up and read Getting Started while waiting.
4.  Run vagrant ssh to SSH into the development machine (remember to `cd /vagrant`).

In case you do not want to use Vagrant, you can install the dependencies from the installation section and run:
To setup Fanboi2 in development mode, run the following commands after performing production setup steps:

    $ make dev
    $ make devhook

You can then configure the application (see configuration section) and run the server:
And run the server with (which will run everything required for development):

    $ make migrate
    $ make devrun

### Docker
### FreeBSD

[Fanboi Channel](https://fanboi.ch/) uses FreeBSD as its deploy target. We no longer provides Vagrantfile due to complexity to maintain the environment, however in case a FreeBSD VM is used (e.g. via Bhyve, Xhyve, Virtualbox or other virtual machine applications) we provide a `scripts/bootstrap.sh` script which should setup the environment to be as close to the production setup as much as possible. To use the script:

1. Create a virtual machine running [FreeBSD 12.0-RELEASE](https://www.freebsd.org/releases/12.0R/relnotes.html)
2. Setup your user with `sudo` and SSH.
3. Run `cat builds/bootstrap.sh | ssh user@host sudo sh`
4. Configure NFS mount or clone the project according to the instruction above.

### Docker Compose

To ease the development, we also provide a [Docker Compose](https://docs.docker.com/compose/) file suitable for both development and evaluation purpose. Please note that the resulting Dockerfile is not being used in [Fanboi Channel](https://fanboi.ch/) and may regress from time to time. To use this `docker-compose.yml` for evaluation purpose, simply use the auto-configuration tool:

    $ ./docker-gen.sh -o docker-compose.override.yml
    $ docker-compose up -d

In case you wish to develop using Docker Compose, instead run:

1.  Install `Docker` and `Docker Compose`
2.  Copy Compose configuration files to the application's parent directory (`cp vendor/docker/docker-compose.* ../`)
3.  Modify the content of `docker-compose.yml`
    1.  Generate both `AUTH_SECRET` and `SESSION_SECRET` tokens with `openssl rand -hex 32`
    2.  Set a sensible PostgreSQL password
    3.  This config assumes fanboi2 was cloned to `fanboi2`; update the build and mount paths if untrue
4.  Start the contraption with `docker-compose up` from the same directory as the config files.
    $ ./docker-gen.sh -d -o docker-compose.override.yml
    $ docker-compose up --build -d

Images are published to [sirn/fanboi2](https://hub.docker.com/r/sirn/fanboi2/) on Docker Hub for every commit in master. By default, using Fanboi2 with the default `docker-compose.yml` will start server in development mode which aids debugging. To disable development server capabilities, remove or rename the file `docker-compose.override.yml`.
Please inspect and adjust `docker-compose.override.yml` as needed.

### Submitting changes



@@ 171,7 172,7 @@ Also install `pre-commit-hook` if you want to contribute to the project:

## License

Copyright © 2013-2018, Kridsada Thanabulpong. All rights reserved.
Copyright © 2013-2019, Kridsada Thanabulpong. All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:


D Vagrantfile => Vagrantfile +0 -23
@@ 1,23 0,0 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
  config.vm.define "web", primary: true do |web|
    web.vm.box = "generic/freebsd11"
    web.vm.network "private_network", ip: "10.200.80.100"
    web.vm.network "forwarded_port", guest: 6543, host: 6543
    web.vm.synced_folder ".", "/vagrant", type: "nfs", mount_options: ["actimeo=2"]
    web.ssh.shell = "sh"
    web.vm.provision :shell, privileged: true, path: "vendor/vagrant/bootstrap_web.sh"
  end

  # The builder vm is for building Docker image in case you're not a fan of Docker.
  # By default it's not created when running `vagrant up`.
  # To use it, explicitly run `vagrant up builder`.
  config.vm.define "builder", autostart: false do |builder|
    builder.vm.box = "generic/debian9"
    builder.vm.network "private_network", ip: "10.200.80.101"
    builder.vm.synced_folder ".", "/vagrant", type: "nfs", mount_options: ["actimeo=2"]
    builder.vm.provision :shell, privileged: true, path: "vendor/vagrant/bootstrap_builder.sh"
  end
end

A docker-compose.override.yml => docker-compose.override.yml +59 -0
@@ 0,0 1,59 @@
---
version: "3.4"

x-presets:
  fanboi2: &fanboi2
    environment:
      AUTH_SECRET: 1c2e2a936a56382159bdceecc4526b1bbc01ceffb05b6ba7318d346e208d1243
      SESSION_SECRET: 28f2c6cb686e4eae10db6fb05626591ba5972a16f1aa0daf83bd9f563aaafd06
      DATABASE_URL: postgresql://fanboi2:H-DK6fcehRQpTMTh2_U9yiJuco8YtQl1@postgres:5432/fanboi2
      POSTGRESQL_TEST_DATABASE: postgresql://fanboi2:H-DK6fcehRQpTMTh2_U9yiJuco8YtQl1@postgres:5432/fanboi2_test
      SERVER_DEV: "true"
    image: sirn/fanboi2:dev
    volumes:
      - ./Makefile:/src/Makefile
      - ./alembic.ini:/src/alembic.ini
      - ./fanboi2:/src/fanboi2
      - ./migration:/src/migration
      - ./setup.cfg:/src/setup.cfg
      - ./setup.py:/src/setup.py

services:
  postgres:
    environment:
      DB_PASSWORD: H-DK6fcehRQpTMTh2_U9yiJuco8YtQl1

  web:
    <<: *fanboi2
    build: .
    command: [make, devserve]

  assets:
    image: sirn/fanboi2-assets:dev
    build:
      context: .
      target: assets
    volumes:
      - ./Makefile:/src/Makefile
      - ./assets:/src/assets
      - ./fanboi2/static:/src/fanboi2/static
      - ./gulpfile.js:/src/gulpfile.js
      - ./package.json:/src/package.json
      - ./tsconfig.json:/src/tsconfig.json
      - ./yarn.lock:/src/yarn.lock
    command: [make, devassets]

  worker:
    <<: *fanboi2
    depends_on:
      - web

  beat:
    <<: *fanboi2
    depends_on:
      - web

  migrate:
    <<: *fanboi2
    depends_on:
      - web

A docker-compose.yml => docker-compose.yml +73 -0
@@ 0,0 1,73 @@
---
version: "3.4"

x-presets:
  fanboi2: &fanboi2
    image: sirn/fanboi2:latest
    restart: always
    networks:
      - fbnet
    depends_on:
      - postgres
      - redis
    environment:
      AUTH_SECRET: CHANGE_ME
      SESSION_SECRET: CHANGE_ME
      DATABASE_URL: postgresql://postgres:passw0rd@postgres:5432/fanboi2
      REDIS_URL: redis://redis/0
      CELERY_BROKER_URL: redis://redis/1

services:
  postgres:
    image: postgres:10.5-alpine
    restart: always
    volumes:
      - postgres-db:/var/lib/postgresql/data
      - ./scripts/initdb.sh:/docker-entrypoint-initdb.d/initdb.sh
    networks:
      - fbnet
    environment:
      DB_PASSWORD: passw0rd
    healthcheck:
      test: [CMD-SHELL, pg_isready, -U, postgres]
      interval: 30s
      retries: 3

  redis:
    image: redis:5.0-alpine
    restart: always
    volumes:
      - redis-db:/data
    networks:
      - fbnet
    healthcheck:
      test: [CMD-SHELL, redis-cli, ping]
      interval: 30s
      retries: 3

  web:
    <<: *fanboi2
    command:
      - serve
    ports:
      - "6543:6543"

  worker:
    <<: *fanboi2
    command: [worker]

  beat:
    <<: *fanboi2
    command: [beat]

  migrate:
    <<: *fanboi2
    restart: on-failure
    command: [migrate]

networks:
  fbnet:

volumes:
  postgres-db:
  redis-db:

A docker-gen.sh => docker-gen.sh +190 -0
@@ 0,0 1,190 @@
#!/bin/sh

set -e

## Arguments handling
##

print_usage() {
    printf "Usage: %s [CONFIG...] [OPTS...]\\n" "$0"
    printf "\\n"
    printf "OPTS:\\n"
    printf "\\n"
    printf "     -h                      Print this help.\\n"
    printf "     -d                      Generates a development overrides.\\n"
    printf "     -o FILENAME             Output to the given file.\\n"
    printf "\\n"
    printf "CONFIG:\\n"
    printf "\\n"
    printf "     -i IMAGE                Specify an alternative image name.\\n"
    printf "     -n NETWORK              Specify an external network.\\n"
    printf "     -p DB_PASSWORD          Specify a PostgreSQL password.\\n"
    printf "     -s AUTH_SECRET          Specify an AUTH_SECRET variable.\\n"
    printf "     -S SESSION_SECRET       Specify a SESSION_SECRET variable.\\n"
    printf "     -f ENV                  Specify an ENV file to load ENV from.\\n"
    printf "\\n"
}

OPTIND=1

DEV_MODE=0
OUTPUT_FILENAME=""
IMAGE_NAME=""
EXTERNAL_NETWORK=""
DB_PASSWORD=""
AUTH_SECRET=""
SESSION_SECRET=""
ENV_FILE=""

while getopts "hdo:i:n:p:s:S:f:" opt; do
    case "$opt" in
        d ) DEV_MODE=1;;
        o ) OUTPUT_FILENAME=$OPTARG;;
        i ) IMAGE_NAME=$OPTARG;;
        n ) EXTERNAL_NETWORK=$OPTARG;;
        p ) DB_PASSWORD=$OPTARG;;
        s ) AUTH_SECRET=$OPTARG;;
        S ) SESSION_SECRET=$OPTARG;;
        f ) ENV_FILE=$OPTARG;;
        h ) print_usage; exit 2;;
        * ) print_usage; exit 1;;
    esac
done

shift $((OPTIND-1))
if [ "${1:-}" = "--" ]; then
    shift
fi


## Normalizing
##

[ -z "$SESSION_SECRET" ] && SESSION_SECRET=$(openssl rand -hex 32)
[ -z "$AUTH_SECRET" ]    && AUTH_SECRET=$(openssl rand -hex 32)
[ -z "$DB_PASSWORD" ]    && DB_PASSWORD=$(openssl rand -base64 24 | tr '+/' '-_')
[ -z "$IMAGE_NAME" ]     && IMAGE_NAME="sirn/fanboi2:latest"


## Presets
##

YML_PRESETS="$YML_PRESETS
x-presets:
  fanboi2: &fanboi2
    environment:
      AUTH_SECRET: $AUTH_SECRET
      SESSION_SECRET: $SESSION_SECRET
      DATABASE_URL: postgresql://fanboi2:$DB_PASSWORD@postgres:5432/fanboi2\
" # EOF

if [ $DEV_MODE = 0 ]; then
    YML_PRESETS="$YML_PRESETS
    image: $IMAGE_NAME\
" # EOF
else
    YML_PRESETS="$YML_PRESETS
      POSTGRESQL_TEST_DATABASE: postgresql://fanboi2:$DB_PASSWORD@postgres:5432/fanboi2_test
      SERVER_DEV: \"true\"
    image: sirn/fanboi2:dev
    volumes:
      - ./Makefile:/src/Makefile
      - ./alembic.ini:/src/alembic.ini
      - ./fanboi2:/src/fanboi2
      - ./migration:/src/migration
      - ./setup.cfg:/src/setup.cfg
      - ./setup.py:/src/setup.py\
" # EOF
fi

if [ -n "$ENV_FILE" ]; then
    YML_PRESETS="$YML_PRESETS
    env_file:
      - $ENV_FILE\
" # EOF
fi


## Services
##

YML_SERVICES="$YML_SERVICES
  postgres:
    environment:
      DB_PASSWORD: $DB_PASSWORD

  web:
    <<: *fanboi2\
" # EOF

if [ $DEV_MODE = 1 ]; then
    YML_SERVICES="$YML_SERVICES
    build: .
    command: [make, devserve]

  assets:
    image: sirn/fanboi2-assets:dev
    build:
      context: .
      target: assets
    volumes:
      - ./Makefile:/src/Makefile
      - ./assets:/src/assets
      - ./fanboi2/static:/src/fanboi2/static
      - ./gulpfile.js:/src/gulpfile.js
      - ./package.json:/src/package.json
      - ./tsconfig.json:/src/tsconfig.json
      - ./yarn.lock:/src/yarn.lock
    command: [make, devassets]
\
" # EOF
fi

YML_SERVICES="$YML_SERVICES
  worker:
    <<: *fanboi2
    depends_on:
      - web

  beat:
    <<: *fanboi2
    depends_on:
      - web

  migrate:
    <<: *fanboi2
    depends_on:
      - web\
" # EOF


## Network
##

if [ -n "$EXTERNAL_NETWORK" ]; then
    YML_NETWORKS="
  fbnet:
    external:
      name: $EXTERNAL_NETWORK\
" # EOF
fi


## Output
##

print_yaml() {
    printf -- "---\\nversion: \"3.4\"\\n"
    [ -n "$YML_PRESETS" ]  && printf "%s\\n"             "$YML_PRESETS"
    [ -n "$YML_SERVICES" ] && printf "\\nservices:%s\\n" "$YML_SERVICES"
    [ -n "$YML_NETWORKS" ] && printf "\\nnetworks:%s\\n" "$YML_NETWORKS"
}

if [ -n "$OUTPUT_FILENAME" ]; then
    TMPFILE=$(mktemp)
    trap 'rm -f $TMPFILE' 0 1 2 3 6 14 15
    print_yaml | tee "$TMPFILE" >/dev/null
    cp "$TMPFILE" "$OUTPUT_FILENAME"
else
    print_yaml
fi

M fanboi2/cmd/ctl.py => fanboi2/cmd/ctl.py +16 -16
@@ 24,7 24,7 @@ Loaded locals:
"""


def shell(args):
def run_shell(args):
    """Run the interactive shell for the application."""
    from ..wsgi import app, config
    import pyramid.scripting


@@ 36,7 36,7 @@ def shell(args):
        )


def serve(args):
def run_serve(args):
    """Run the web server for the application."""
    from gunicorn.app.base import BaseApplication
    from ..wsgi import app as wsgi_app


@@ 68,7 68,7 @@ def serve(args):
    FbserveApplication(wsgi_app, options).run()


def gensecret(args):
def run_gensecret(args):
    """Generates a NaCl secret."""
    from pyramid_nacl_session import generate_secret



@@ 81,21 81,21 @@ def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(dest="subparser", title="subcommands")

    pshell = subparsers.add_parser("shell")
    pshell.set_defaults(func=shell)
    shell = subparsers.add_parser("shell")
    shell.set_defaults(func=run_shell)

    cpu_count = multiprocessing.cpu_count()
    pserve = subparsers.add_parser("serve")
    pserve.add_argument("--port", type=int, default=6543)
    pserve.add_argument("--host", default="0.0.0.0")
    pserve.add_argument("--workers", type=int, default=(cpu_count * 2) + 1)
    pserve.add_argument("--threads", type=int, default=1)
    pserve.add_argument("--max-requests", type=int, default=1000)
    pserve.add_argument("--reload", action="store_true")
    pserve.set_defaults(func=serve)

    gsecret = subparsers.add_parser("gensecret")
    gsecret.set_defaults(func=gensecret)
    serve = subparsers.add_parser("serve")
    serve.add_argument("--port", type=int, default=6543)
    serve.add_argument("--host", default="0.0.0.0")
    serve.add_argument("--workers", type=int, default=(cpu_count * 2) + 1)
    serve.add_argument("--threads", type=int, default=1)
    serve.add_argument("--max-requests", type=int, default=1000)
    serve.add_argument("--reload", action="store_true")
    serve.set_defaults(func=run_serve)

    gensecret = subparsers.add_parser("gensecret")
    gensecret.set_defaults(func=run_gensecret)

    args = parser.parse_args()
    if args.subparser is None:

R vendor/rootfs/entrypoint => rootfs/entrypoint +2 -1
@@ 5,4 5,5 @@ s6-setuidgid fanboi2
ifelse { test $1 == "serve" } { make serve }
ifelse { test $1 == "worker" } { make worker }
ifelse { test $1 == "migrate" } { make migrate }
exec $@
\ No newline at end of file
ifelse { test $1 == "beat" } { make beat }
exec $@

A scripts/bootstrap.sh => scripts/bootstrap.sh +247 -0
@@ 0,0 1,247 @@
#!/bin/sh

## Utils
##

echo_clear() {
    printf "\\033[1A\\r\\033[K"
}

echo_newline() {
    printf "\\n"
}

echo_ok() {
    printf "=> \\033[1;37m%s\\033[0;0m\\n" "$1"
}

echo_wait() {
    printf "=> \\033[0;33m%s\\033[0;0m\\n" "$1"
}

echo_error() {
    printf "=> \\033[0;31m%s\\033[0;0m\\n" "$1"
}

echo_info() {
    printf "   \\033[0;36m%s\\033[0;0m\\n" "$1"
}


## Sanity check
##

echo_wait "Checking system..."

if [ "$(uname)" != "FreeBSD" ]; then
    echo_error "Development environment setup script only supports FreeBSD."
    echo_info "For other operating systems, please see README.md."
    exit 1
fi

if [ "$(id -u)" != "0" ] || [ -z "$SUDO_USER" ]; then
    echo_error "Must be run with sudo."
    echo_info "If this was run over SSH, the correct command should be:"
    echo_info "cat builds/bsd-bootstrap.sh | ssh host sudo sh"
    exit 1
fi

LOGIN_USER=$SUDO_USER


## Ports
##

if [ -f "/usr/ports/Makefile" ]; then
    echo_ok "FreeBSD Ports already exists."
else
    echo_wait "Development setup script requires FreeBSD Ports. Fetching..."
    portsnap auto
fi

if hash synth 2>&1; then
    echo_ok "Synth is already installed."
else
    echo_wait "Synth is not installed. Installing..."
    make -C /usr/ports/ports-mgmt/synth install clean
fi


## Installation
##

mkdir -p /usr/local/etc/synth/
echo_wait "Configuring Synth..."

if [ -f /usr/local/etc/synth/LiveSystem-make.conf ]; then
    echo_info "Configuration for LiveSytem already exists at /usr/local/etc/synth."
    echo_info "Moving LiveSystem-make.conf into LiveSystem-make.conf.bak"
    mv /usr/local/etc/synth/LiveSystem-make.conf /usr/local/etc/synth/LiveSystem-make.conf.bak
fi

cat <<EOF > /usr/local/etc/synth/LiveSystem-make.conf
DEFAULT_VERSIONS+=ssl=libressl

# ftp/curl
ftp_curl_UNSET+=GSSAPI_NONE TLS_SRP
ftp_curl_SET+=GSSAPI_HEIMDAL

# security/p5-GSSAPI <- ftp/curl
security_p5-GSSAPI_UNSET+=GSSAPI_BASE
security_p5-GSSAPI_SET+=GSSAPI_HEIMDAL

# www/node8
www_node8_SET+=BUNDLED_SSL

# www/yarn
www_yarn_UNSET+=NODE
www_yarn_SET+=NODE8
EOF

INSTALL_LIST="$(mktemp)"
trap 'rm $INSTALL_LIST' 0 1 2 3 6 14 15

cat <<EOF > "$INSTALL_LIST"
databases/postgresql10-server
databases/py-sqlite3@py36
databases/redis
databases/sqlite3
devel/git-lite
devel/py-pip@py36
devel/py-virtualenv@py36
ftp/curl
graphics/GraphicsMagick
lang/python36
net/openntpd
security/ca_root_nss
www/node8
www/npm-node8
www/yarn
EOF

echo_wait "Installing development dependencies..."
echo_info "This may take a (really) long time."

# Redirecting STDIN otherwise Synth will cause the rest of
# shell script to broke for some reason...
synth install "$INSTALL_LIST" </dev/null


## Enabling services
##

if service openntpd onestatus >/dev/null; then
    echo_ok "OpenNTPd is already enabled."
else
    echo_wait "Enabling OpenNTPd..."
    sysrc openntpd_enable=YES
    service openntpd start
fi

if service postgresql onestatus >/dev/null; then
    echo_ok "PostgreSQL is already enabled."
else
    echo_wait "Enabling PostgreSQL..."
    service postgresql oneinitdb

    # Note: pg_ctl will write its output to controlling terminal which
    # will cause shell process to never terminate. See also pg_ctl(1).
    sysrc postgresql_enable=YES
    sysrc postgresql_flags="-l /var/log/postgresql.log"

    touch /var/log/postgresql.log
    chown postgres:postgres /var/log/postgresql.log

    cat <<EOF > /var/db/postgres/data10/pg_hba.conf
local all all trust
host all all 127.0.0.1/32 trust
host all all ::1/128 trust
EOF

    service postgresql start
    sudo -u postgres createuser -ds "$LOGIN_USER" || true
    sudo -u postgres createuser -ds fanboi2 || true
fi

if service redis onestatus >/dev/null; then
    echo_ok "Redis is already enabled."
else
    echo_wait "Enabling Redis..."
    sysrc redis_enable=YES
    service redis start
fi

# nfsclient doesn't provide status so we're checking
# the lock daemon instead.
if service lockd onestatus >/dev/null; then
    echo_ok "NFS is already enabled."
else
    echo_wait "Enabling NFS..."
    sysrc nfs_client_enable=YES
    sysrc rpc_lockd_enable="YES"
    sysrc rpc_statd_enable="YES"
    service nfsclient start
    service lockd start
    service statd start
fi


## User configurations
##

echo_wait "Setting up user environment..."
echo_info "You must setup NFS mount and run \`make devserver\` on your own."
echo_info "However initial configuration will be provided at .local/fanboi2/env"
echo_info "with ENVFILE configured."

sudo -u "$LOGIN_USER" sh <<EOF
cd "\$HOME" || exit
rm -f \
   .bashrc \
   .cshrc \
   .lesshst \
   .login \
   .login-e \
   .login_conf \
   .mail_aliases \
   .mailrc \
   .profile \
   .profile-e \
   .rhosts \
   .rnd \
   .shrc

cat <<EOPROFILE > "\$HOME/.profile"
EDITOR=vim; export EDITOR
PAGER=more; export PAGER
LANG=en_US.UTF-8; export LANG

# Fanboi2
FBVAR=\$HOME/.local/fanboi2; export FBVAR
VIRTUALENV=virtualenv-3.6; export VIRTUALENV
BUILDDIR=\\\$FBVAR/build; export BUILDDIR
VENVDIR=\\\$FBVAR/venv; export VENVDIR
ENVFILE=\\\$FBVAR/env; export ENVFILE
PATH=\\\$VENVDIR/bin:\\\$PATH; export PATH
EOPROFILE

. "\$HOME/.profile"
mkdir -p \$FBVAR

cat <<EOENV > "\$ENVFILE"
CELERY_BROKER_URL=redis://127.0.0.1:6379/1
DATABASE_URL=postgresql://fanboi2:@127.0.0.1:5432/fanboi2_dev
POSTGRESQL_TEST_DATABASE=postgresql://fanboi2:@127.0.0.1:5432/fanboi2_test
REDIS_URL=redis://127.0.0.1:6379/0
SERVER_DEV=true
SERVER_HOST=0.0.0.0
SERVER_PORT=6543
SESSION_SECRET=\$(openssl rand -hex 32)
AUTH_SECRET=\$(openssl rand -hex 32)
EOENV
EOF

(
    psql -U fanboi2 template1 -c "CREATE DATABASE fanboi2_dev;" || true
    psql -U fanboi2 template1 -c "CREATE DATABASE fanboi2_test;" || true
) >/dev/null 2>&1

A scripts/initdb.sh => scripts/initdb.sh +16 -0
@@ 0,0 1,16 @@
#!/bin/sh

set -e

if [ -z "$DB_PASSWORD" ]; then
    printf "DB_PASSWORD is not present, exiting.\\n"
    exit 1
fi

psql -v ON_ERROR_STOP=1 --username="$POSTGRES_USER" --dbname="$POSTGRES_DB" <<EOF
CREATE USER fanboi2 WITH ENCRYPTED PASSWORD '$DB_PASSWORD';
CREATE DATABASE fanboi2 WITH OWNER fanboi2;
CREATE DATABASE fanboi2_test WITH OWNER fanboi2;
GRANT ALL PRIVILEGES ON DATABASE fanboi2 TO fanboi2;
GRANT ALL PRIVILEGES ON DATABASE fanboi2_test TO fanboi2;
EOF

R vendor/builds/utils.sh => scripts/utils.sh +0 -0
M setup.py => setup.py +3 -2
@@ 11,7 11,7 @@ else:

setup(
    name="fanboi2",
    version="2018.12",
    version="2019.02",
    description="Board engine behind fanboi.ch",
    long_description=LONG_DESCRIPTION,
    long_description_content_type="text/markdown",


@@ 43,6 43,7 @@ setup(
        "gunicorn",
        "hiredis >=0.2, <0.3",
        "isodate",
        "kombu >= 4.3, <4.4",
        "lark-parser >=0.6, <0.7",
        "misaka",
        "passlib",


@@ 64,7 65,7 @@ setup(
    zip_safe=False,
    test_suite="fanboi2.tests",
    extras_require={
        "dev": ["honcho", "pre-commit"],
        "dev": ["honcho", "hupper", "pre-commit"],
        "test": ["nose", "coverage", "rednose"],
        "deploy": ["fabric", "patchwork", "invocations", "colorama"],
    },

D vendor/docker/README.rst => vendor/docker/README.rst +0 -25
@@ 1,25 0,0 @@
============================
Docker Compose Configuration
============================

These configuration are intended to help you launch Fanboi2 and all its dependencies on one machine, mainly for development/staging. For a real production deployment, you will want to use container orchestration tools such as Docker Swarm or Kubernetes, which are beyond the scope of this document.

Setting Up
----------

These configuration files assume the following directory structure::

  project_root/
  ├── docker-compose.yml
  ├── docker-compose.override.yml
  └── fanboi2/
      └── <REPOSITORY CONTENT>

1. Create the above directory structure by copying both `yml` files to the same level as the repository directory
2. Edit the content of `docker-compose.yml` and initialize the variables with freshly generated secret tokens
3. Run `docker-compose build && docker-compose up -d` from the context of `project_root` (*not* the repository root)

Disabling Code Reload
---------------------

Removing or renaming `docker-compose.override.yml` to something else will prevent automatic reloading and rebuilding of code, similar to when in production. Don't forget to re-run `docker-compose up -d` when you make any changes.

D vendor/docker/docker-compose.override.yml => vendor/docker/docker-compose.override.yml +0 -15
@@ 1,15 0,0 @@
version: '2.3'
services:

  web:
    environment:
      SERVER_DEV: "true"
    command: devserve

  assets:
    build:
      context: ./fanboi2
      target: assets
    volumes:
      - ./fanboi2:/src
    command: make devassets

D vendor/docker/docker-compose.yml => vendor/docker/docker-compose.yml +0 -39
@@ 1,39 0,0 @@
version: '2.3'

x-app:
  &app-defaults
  build: ./fanboi2
  volumes:
    - ./fanboi2:/src
  links:
    - postgres
    - redis
  environment:
    AUTH_SECRET: <REPLACE WITH SECRET>
    SESSION_SECRET: <REPLACE WITH SECRET>
    DATABASE_URL: postgres://postgres:<REPLACE WITH PASSWORD>@postgres/fanboi2
    REDIS_URL: redis://redis/0
    CELERY_BROKER_URL: redis://redis/1

services:
  web:
    <<: *app-defaults
    ports:
      - '6543:6543'

  worker:
    <<: *app-defaults
    command: worker

  migrations:
    <<: *app-defaults
    command: migrate

  postgres:
    image: postgres:10.5-alpine
    environment:
      POSTGRES_PASSWORD: <REPLACE WITH PASSWORD>
      POSTGRES_DB: fanboi2

  redis:
    image: redis:5.0-alpine

D vendor/vagrant/bootstrap_builder.sh => vendor/vagrant/bootstrap_builder.sh +0 -47
@@ 1,47 0,0 @@
#!/bin/sh
set -xe

apt-get update
apt-get install -y \
        apt-transport-https \
        ca-certificates \
        curl \
        dirmngr \
        gnupg2 \
        software-properties-common

#
# Docker
#

curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
apt-get update
apt-get install -y docker-ce docker-compose

systemctl enable docker
systemctl start docker
usermod -aG docker vagrant

#
# IMG
#

apt-get update
apt-get install -y libseccomp-dev uidmap
echo 1 > /proc/sys/kernel/unprivileged_userns_clone

_img_sha256="6b7b660fa0a4c4ab10aa2c2d7d586afdbc70cb33644995b0ee0e7f77ddcc2565"
_img_version="v0.5.4"
curl -fSL "https://github.com/genuinetools/img/releases/download/$_img_version/img-linux-amd64" -o "/usr/local/bin/img" \
    && echo "${_img_sha256}  /usr/local/bin/img" | sha256sum -c - \
    && chmod a+x "/usr/local/bin/img"

#
# Google Cloud SDK
#

curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
apt-add-repository "deb [arch=amd64] http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main"
apt-get update
apt-get install -y google-cloud-sdk

D vendor/vagrant/bootstrap_web.sh => vendor/vagrant/bootstrap_web.sh +0 -91
@@ 1,91 0,0 @@
#!/bin/sh
set -xe

sysrc hostname=vagrant
hostname vagrant

# generic/freebsd11 image ship with broken rc.conf, i.e. use `firstboot-growfs`
# instead of `firstboot_growfs`, causing lots of warnings when running pkg.
if awk '! /^firstboot/' < /etc/rc.conf > /etc/rc.conf.tmp; then
    mv /etc/rc.conf.tmp /etc/rc.conf
fi

pkg update -qf

PACKAGES="bzip2 ca_root_nss curl git-lite gmake ntp sqlite3 sudo"
PACKAGES="$PACKAGES postgresql10-server redis"
PACKAGES="$PACKAGES python36 py36-pip py36-sqlite3 py36-virtualenv"
PACKAGES="$PACKAGES node8 npm-node8"

# shellcheck disable=SC2086
pkg install -qy $PACKAGES
npm install -g yarn

if ! service ntpd onestatus >/dev/null; then
    sysrc ntpd_enable=YES
    ntpd -qg
    service ntpd start
fi

if ! service postgresql onestatus >/dev/null; then
    sysrc postgresql_enable=YES

    service postgresql initdb
    cat <<EOF > /var/db/postgres/data10/pg_hba.conf
local all all trust
host all all 127.0.0.1/32 trust
host all all ::1/128 trust
EOF

    service postgresql start
    sudo -u postgres createuser -ds vagrant || true
    sudo -u postgres createuser -ds fanboi2 || true
fi

if ! service redis onestatus >/dev/null; then
    sysrc redis_enable=YES
    service redis start
fi

chsh -s /bin/sh vagrant
sudo -u vagrant sh <<EOF
cd "\$HOME" || exit
rm -f \
   .bashrc \
   .cshrc \
   .lesshst \
   .login \
   .login-e \
   .login_conf \
   .mail_aliases \
   .mailrc \
   .profile \
   .profile-e \
   .rhosts \
   .rnd \
   .shrc

cat <<EOPROFILE > "\$HOME/.profile"
EDITOR=vim; export EDITOR
PAGER=more; export PAGER
LANG=en_US.UTF-8; export LANG
EOPROFILE

cd /vagrant || exit

psql template1 -c "CREATE DATABASE fanboi2_dev;"
psql template1 -c "CREATE DATABASE fanboi2_test;"

cat <<EOENV > "/vagrant/.env"
CELERY_BROKER_URL=redis://127.0.0.1:6379/1
DATABASE_URL=postgresql://vagrant:@127.0.0.1:5432/fanboi2_dev
REDIS_URL=redis://127.0.0.1:6379/0
SERVER_DEV=true
SERVER_HOST=0.0.0.0
SERVER_PORT=6543
SESSION_SECRET=\$(openssl rand -hex 32)
AUTH_SECRET=\$(openssl rand -hex 32)
EOENV

VIRTUALENV=virtualenv-3.6 make -j$(sysctl -n hw.ncpu) assets dev
EOF