~mna/tulip-cli

1b6a42960f10ecf47d76cb5d2f4c950cd7dd5f21 — Martin Angers 1 year, 10 months ago b4e0b77
Implement the db reset and db clean commands
M rockspecs/tulip-cli-0.0.1-1.rockspec => rockspecs/tulip-cli-0.0.1-1.rockspec +1 -0
@@ 16,6 16,7 @@ dependencies = {
   "luaposix 35.0-1",
   "luashell 0.4-1",
   "optparse 1.4-1",
   "xpgsql 0.5-1",
}
build = {
   type = "builtin",

A src/cmds/dbcmd.lua => src/cmds/dbcmd.lua +155 -0
@@ 0,0 1,155 @@
local utils = require 'src.utils'
local xpgsql = require 'xpgsql'

local SQL_TESTDB = [[
SELECT
  datname
FROM
  pg_database
WHERE
  (NOT datistemplate) AND
  datname LIKE '%s%%'
]]

local SQL_DROPDB = [[
DROP DATABASE %s WITH (FORCE)
]]

local SQL_TESTUSER = [[
SELECT
  usename
FROM
  pg_user
WHERE
  (NOT usesuper) AND
  usename LIKE '%s%%'
]]

local SQL_DROPUSER = [[
DROP USER %s
]]

local SQL_CREATEDB = [[
CREATE DATABASE %s
]]

local SQL_CREATECRON = [[
CREATE EXTENSION pg_cron
]]

local SQL_CREATEPLLUA = [[
CREATE EXTENSION pllua
]]

local function dbreset(args, opts, errf)
  local _, _ = args, errf

  local maindb = os.getenv('PGDATABASE')
  local tempdb = 'test' .. maindb .. utils.random_string(12)

  utils.log([[
> about to reset database %s, all data will be lost

Press ENTER to start.
]], maindb)
  if not opts.yes then
    io.read('l')
  end

  -- first, connect to the current maindb to create the temporary db
  local conn = assert(xpgsql.connect())
  assert(conn:with(true, function()
    utils.log('> creating temporary database to connect to...')
    assert(conn:exec(string.format(SQL_CREATEDB, tempdb)))
    utils.log(' ok\n')

    return true
  end))

  -- now connect to the tempdb and drop the main and re-create it
  conn = assert(xpgsql.connect('dbname=' .. tempdb))
  assert(conn:with(true, function()
    utils.log('> re-creating database %s...', maindb)
    assert(conn:exec(string.format(SQL_DROPDB, maindb)))
    assert(conn:exec(string.format(SQL_CREATEDB, maindb)))
    utils.log(' ok\n')
    return true
  end))

  -- and finally, re-connect to the main and drop the temp
  conn = assert(xpgsql.connect())
  assert(conn:with(true, function()
    utils.log('> dropping the temporary database...')
    assert(conn:exec(string.format(SQL_DROPDB, tempdb)))
    utils.log(' ok\n')

    utils.log('> re-creating the extensions...')
    assert(conn:exec(SQL_CREATECRON))
    assert(conn:exec(SQL_CREATEPLLUA))
    utils.log(' ok\n')
    return true
  end))
  utils.log('> reset completed successfully.\n')
end

local function dbclean(args, opts, errf)
  local _ = args

  if not opts.prefix then
    errf('the --prefix option is required')
    return
  end
  local prefix = opts.prefix

  local conn = assert(xpgsql.connect())
  assert(conn:with(true, function()
    local dbs = xpgsql.models(assert(conn:query(string.format(SQL_TESTDB, prefix))))

    utils.log([[
> about to delete %d matching databases

Press ENTER to start.
]], #dbs)
    if not opts.yes then
      io.read('l')
    end

    for _, db in ipairs(dbs) do
      assert(conn:exec(string.format(SQL_DROPDB, db.datname)))
    end
    utils.log('> deleted %d databases\n', #dbs)

    local users = xpgsql.models(assert(conn:query(string.format(SQL_TESTUSER, prefix))))
    utils.log('> deleting %d users...', #users)
    for _, user in ipairs(users) do
      assert(conn:exec(string.format(SQL_DROPUSER, user.usename)))
    end
    utils.log(' ok\n')

    return true
  end))
  utils.log('> clean completed successfully.\n')
end

local subcmds = {
  clean = dbclean,
  reset = dbreset,
}

return function(args, opts, errf)
  if #args == 0 then
    errf('the db sub-command is required')
    return
  elseif #args > 1 then
    errf('unexpected argument starting at %q', args[2])
    return
  end

  local subcmd = table.remove(args, 1)
  local f = subcmds[subcmd]
  if not f then
    errf('unknown db sub-command %q', subcmd)
    return
  end
  return f(args, opts, errf)
end

A src/cmds/files/docker_compose.lua => src/cmds/files/docker_compose.lua +20 -0
@@ 0,0 1,20 @@
return [[
version: '3.6'
services:
  pg:
    build: ./db/postgres/image
    command: -c 'config_file=/etc/postgresql/postgresql.conf'
    ports:
      - "127.0.0.1:%d:5432"
    environment:
      - POSTGRES_PASSWORD_FILE=/run/secrets/root_pwd
    volumes:
      - ./db/postgres/config/postgres.conf:/etc/postgresql/postgresql.conf:ro
      - ./db/postgres/data:/var/lib/postgresql/data
    secrets:
      - root_pwd

secrets:
  root_pwd:
    file: ./run/secrets/pgroot_pwd
]]

A src/cmds/files/dockerfile.lua => src/cmds/files/dockerfile.lua +30 -0
@@ 0,0 1,30 @@
return [[
FROM postgres:13

RUN set -ex                      \
      && apt-get --yes update    \
      && apt-get --yes install   \
        build-essential          \
        git                      \
        liblua5.3-dev            \
        lua5.3                   \
        postgresql-server-dev-13 \
      && cd /tmp                 \
      && git clone https://github.com/citusdata/pg_cron.git \
      && cd pg_cron              \
      && make                    \
      && make install            \
      && cd /tmp                 \
      && git clone https://github.com/pllua/pllua-ng.git    \
      && cd pllua-ng             \
      && make                    \
        PG_CONFIG=pg_config      \
        LUA_INCDIR=/usr/include/lua5.3 \
        LUALIB=-llua5.3          \
        LUAC=luac5.3             \
        LUA=lua5.3               \
        install

COPY ./install_pg_cron.sql /docker-entrypoint-initdb.d/install_pg_cron.sql
COPY ./install_pllua.sql   /docker-entrypoint-initdb.d/install_pllua.sql
]]

A src/cmds/files/envrc.lua => src/cmds/files/envrc.lua +13 -0
@@ 0,0 1,13 @@
return [[
export LUA_VERSION=`lua -e 'print(dofile(".luarocks/default-lua-version.lua"))'`
export LUA_MODULES=`pwd`/lua_modules
export LUA_PATH="${LUA_MODULES}/share/lua/${LUA_VERSION}/?.lua;${LUA_MODULES}/share/lua/${LUA_VERSION}/?/init.lua;;"
export LUA_CPATH="${LUA_MODULES}/lib/lua/${LUA_VERSION}/?.so;;"

export PGPASSFILE=`pwd`/run/secrets/pgpass
export PGHOST=localhost
export PGPORT=%d
export PGCONNECT_TIMEOUT=10
export PGUSER=postgres
export PGDATABASE=postgres
]]

A src/cmds/files/gitignore.lua => src/cmds/files/gitignore.lua +19 -0
@@ 0,0 1,19 @@
return [[
# environment files
.env*

# locally-installed lua modules
/lua_modules/

# local luarocks configuration
/.luarocks/

# local runtime directory
/run/

# output files generated by tools
*.out

# rocks packages
*.src.rock
]]

A src/cmds/files/postgres_conf.lua => src/cmds/files/postgres_conf.lua +47 -0
@@ 0,0 1,47 @@
return [[
# See https://github.com/postgres/postgres/blob/master/src/backend/utils/misc/postgresql.conf.sample
#
# -----------------------------
# PostgreSQL configuration file
# -----------------------------
#
# This file consists of lines of the form:
#
#   name = value
#
# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
# "#" anywhere on a line.  The complete list of parameter names and allowed
# values can be found in the PostgreSQL documentation.
#
# The commented-out settings shown in this file represent the default values.
# Re-commenting a setting is NOT sufficient to revert it to the default value;
# you need to reload the server.
#
# This file is read on server startup and when the server receives a SIGHUP
# signal.  If you edit the file on a running system, you have to SIGHUP the
# server for the changes to take effect, run "pg_ctl reload", or execute
# "SELECT pg_reload_conf()".  Some parameters, which are marked below,
# require a server shutdown and restart to take effect.
#
# Any parameter can also be given as a command-line option to the server, e.g.,
# "postgres -c log_connections=on".  Some parameters can be changed at run time
# with the "SET" SQL command.
#
# Memory units:  kB = kilobytes        Time units:  ms  = milliseconds
#                MB = megabytes                     s   = seconds
#                GB = gigabytes                     min = minutes
#                TB = terabytes                     h   = hours
#                                                   d   = days

# - Connection Settings -

listen_addresses = '*'
					# comma-separated list of addresses;
					# defaults to 'localhost'; use '*' for all
					# (change requires restart)

timezone = 'UTC'

shared_preload_libraries = 'pg_cron, pllua'
cron.database_name = 'postgres'
]]

M src/cmds/init.lua => src/cmds/init.lua +1 -0
@@ 1,3 1,4 @@
return {
  db = require 'src.cmds.dbcmd',
  init = require 'src.cmds.initcmd',
}

M src/cmds/initcmd.lua => src/cmds/initcmd.lua +45 -196
@@ 1,4 1,3 @@
local base64 = require 'base64'
local dirent = require 'posix.dirent'
local errno = require 'posix.errno'
local libgen = require 'posix.libgen'


@@ 6,156 5,36 @@ local posix = require 'posix'
local sh = require 'shell'
local stdlib = require 'posix.stdlib'
local unistd = require 'posix.unistd'

local rand = assert(io.open('/dev/urandom', 'r'))
local enc = base64.makeencoder('.', '_', '-')

local function random_string(len)
  local raw = rand:read(len)
  return base64.encode(raw, enc)
end

local function log(s, ...)
  local msg = string.format(s, ...)
  io.write(msg)
  if not string.match(msg, '\n$') then
    io.flush()
  end
end

local function write_file(path, content)
  local fd = assert(io.open(path, 'w+'))
  assert(fd:write(content))
  assert(fd:close())
end
local utils = require 'src.utils'

local function create_luarocks(root_dir, luaver)
  local dir = root_dir .. '/.luarocks'
  assert(posix.mkdir(dir))
  write_file(dir .. string.format('/config-%s.lua', luaver), '')
  write_file(dir .. '/default-lua-version.lua', string.format('return "%s"', luaver))
  utils.write_file(dir .. string.format('/config-%s.lua', luaver), '')
  utils.write_file(dir .. '/default-lua-version.lua', string.format('return "%s"', luaver))
  assert(posix.mkdir(root_dir .. '/lua_modules'))
end

local function create_postgres(root_dir)
local function create_postgres(root_dir, port)
  assert(posix.mkdir(root_dir .. '/db'))
  assert(posix.mkdir(root_dir .. '/db/postgres'))
  assert(posix.mkdir(root_dir .. '/db/postgres/config'))
  assert(posix.mkdir(root_dir .. '/db/postgres/data'))
  assert(posix.mkdir(root_dir .. '/db/postgres/image'))

  write_file(root_dir .. '/db/postgres/.gitignore', [[
  utils.write_file(root_dir .. '/db/postgres/.gitignore', [[
# ignore the data directory
/data/
]])

  write_file(root_dir .. '/db/postgres/config/postgres.conf', [[
# See https://github.com/postgres/postgres/blob/master/src/backend/utils/misc/postgresql.conf.sample
#
# -----------------------------
# PostgreSQL configuration file
# -----------------------------
#
# This file consists of lines of the form:
#
#   name = value
#
# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
# "#" anywhere on a line.  The complete list of parameter names and allowed
# values can be found in the PostgreSQL documentation.
#
# The commented-out settings shown in this file represent the default values.
# Re-commenting a setting is NOT sufficient to revert it to the default value;
# you need to reload the server.
#
# This file is read on server startup and when the server receives a SIGHUP
# signal.  If you edit the file on a running system, you have to SIGHUP the
# server for the changes to take effect, run "pg_ctl reload", or execute
# "SELECT pg_reload_conf()".  Some parameters, which are marked below,
# require a server shutdown and restart to take effect.
#
# Any parameter can also be given as a command-line option to the server, e.g.,
# "postgres -c log_connections=on".  Some parameters can be changed at run time
# with the "SET" SQL command.
#
# Memory units:  kB = kilobytes        Time units:  ms  = milliseconds
#                MB = megabytes                     s   = seconds
#                GB = gigabytes                     min = minutes
#                TB = terabytes                     h   = hours
#                                                   d   = days

# - Connection Settings -

listen_addresses = '*'
					# comma-separated list of addresses;
					# defaults to 'localhost'; use '*' for all
					# (change requires restart)

timezone = 'UTC'

shared_preload_libraries = 'pg_cron, pllua'
cron.database_name = 'postgres'
]])

  write_file(root_dir .. '/db/postgres/image/Dockerfile', [[
FROM postgres:13

RUN set -ex                      \
      && apt-get --yes update    \
      && apt-get --yes install   \
        build-essential          \
        git                      \
        liblua5.3-dev            \
        lua5.3                   \
        postgresql-server-dev-13 \
      && cd /tmp                 \
      && git clone https://github.com/citusdata/pg_cron.git \
      && cd pg_cron              \
      && make                    \
      && make install            \
      && cd /tmp                 \
      && git clone https://github.com/pllua/pllua-ng.git    \
      && cd pllua-ng             \
      && make                    \
        PG_CONFIG=pg_config      \
        LUA_INCDIR=/usr/include/lua5.3 \
        LUALIB=-llua5.3          \
        LUAC=luac5.3             \
        LUA=lua5.3               \
        install

COPY ./install_pg_cron.sql /docker-entrypoint-initdb.d/install_pg_cron.sql
COPY ./install_pllua.sql   /docker-entrypoint-initdb.d/install_pllua.sql
]])

  write_file(root_dir .. '/db/postgres/image/install_pg_cron.sql', [[
  utils.write_file(root_dir .. '/db/postgres/config/postgres.conf', require 'src.cmds.files.postgres_conf')
  utils.write_file(root_dir .. '/db/postgres/image/Dockerfile', require 'src.cmds.files.dockerfile')
  utils.write_file(root_dir .. '/db/postgres/image/install_pg_cron.sql', [[
CREATE EXTENSION pg_cron;
]])

  write_file(root_dir .. '/db/postgres/image/install_pllua.sql', [[
  utils.write_file(root_dir .. '/db/postgres/image/install_pllua.sql', [[
CREATE EXTENSION pllua;
]])

  write_file(root_dir .. '/docker-compose.yml', [[
version: '3.6'
services:
  pg:
    build: ./db/postgres/image
    command: -c 'config_file=/etc/postgresql/postgresql.conf'
    ports:
      - "127.0.0.1:5432:5432"
    environment:
      - POSTGRES_PASSWORD_FILE=/run/secrets/root_pwd
    volumes:
      - ./db/postgres/config/postgres.conf:/etc/postgresql/postgresql.conf:ro
      - ./db/postgres/data:/var/lib/postgresql/data
    secrets:
      - root_pwd

secrets:
  root_pwd:
    file: ./run/secrets/pgroot_pwd
]])
  utils.write_file(root_dir .. '/docker-compose.yml', string.format((require 'src.cmds.files.docker_compose'), port))

  if sh.cmd('command', '-v', 'chcon'):output() then
    -- change the SELinux label for the volume to be mounted on the container


@@ 164,33 43,33 @@ secrets:
  end
end

local function create_secrets(root_dir)
local function create_secrets(root_dir, port)
  local dir = root_dir .. '/run/secrets'
  assert(posix.mkdir(root_dir .. '/run'))
  assert(posix.mkdir(dir))

  -- postgresql root password
  local pgpwd = random_string(32)
  write_file(dir .. '/pgroot_pwd', pgpwd)
  local pgpwd = utils.random_string(32)
  utils.write_file(dir .. '/pgroot_pwd', pgpwd)
  assert(sh('chmod', '0600', dir .. '/pgroot_pwd'))
  if sh.cmd('command', '-v', 'chcon'):output() then
    assert(sh('chcon', '-Rt', 'svirt_sandbox_file_t', dir .. '/pgroot_pwd'))
  end

  -- csrf secret key
  local csrf = random_string(32)
  write_file(dir .. '/csrf_key', csrf)
  local csrf = utils.random_string(32)
  utils.write_file(dir .. '/csrf_key', csrf)
  assert(sh('chmod', '0600', dir .. '/csrf_key'))

  -- account secret key
  local acct = random_string(32)
  write_file(dir .. '/account_key', acct)
  local acct = utils.random_string(32)
  utils.write_file(dir .. '/account_key', acct)
  assert(sh('chmod', '0600', dir .. '/account_key'))

  -- pgpass file
  write_file(dir .. '/pgpass', string.format([[
localhost:5432:*:postgres:%s
]], pgpwd))
  utils.write_file(dir .. '/pgpass', string.format([[
localhost:%d:*:postgres:%s
]], port, pgpwd))
  assert(sh('chmod', '0600', dir .. '/pgpass'))
end



@@ 205,52 84,20 @@ local function create_certs(root_dir)
    'localhost', '127.0.0.1', '::1'))
end

local function create_envrc(root_dir)
  write_file(root_dir .. '/.envrc', [[
export LUA_VERSION=`lua -e 'print(dofile(".luarocks/default-lua-version.lua"))'`
export LUA_MODULES=`pwd`/lua_modules
export LUA_PATH="${LUA_MODULES}/share/lua/${LUA_VERSION}/?.lua;${LUA_MODULES}/share/lua/${LUA_VERSION}/?/init.lua;;"
export LUA_CPATH="${LUA_MODULES}/lib/lua/${LUA_VERSION}/?.so;;"

export PGPASSFILE=`pwd`/run/secrets/pgpass
export PGHOST=localhost
export PGPORT=5432
export PGCONNECT_TIMEOUT=10
export PGUSER=postgres
export PGDATABASE=postgres
]])
local function create_envrc(root_dir, port)
  utils.write_file(root_dir .. '/.envrc', string.format((require 'src.cmds.files.envrc'), port))
end

local function create_gitignore(root_dir)
  write_file(root_dir .. '/.gitignore', [[
# environment files
.env*

# locally-installed lua modules
/lua_modules/

# local luarocks configuration
/.luarocks/

# local runtime directory
/run/

# output files generated by tools
*.out

# rocks packages
*.src.rock
]])
  utils.write_file(root_dir .. '/.gitignore', require 'src.cmds.files.gitignore')
end

local function install_tulip(root_dir)
  assert(unistd.chdir(root_dir))
  assert(sh('luarocks', 'install', 'tulip'))
  assert(sh('luarocks', 'install', '--deps-mode', 'one', 'tulip'))
end

return function(args, opts, errf)
  local _ = opts -- for now, no options available

  if #args == 0 then
    errf('missing directory argument')
    return


@@ 276,44 123,46 @@ return function(args, opts, errf)
  local base = libgen.basename(assert(stdlib.realpath(dir)))
  local luaver = string.match(_VERSION, '(%d%.%d)')

  log([[
  utils.log([[
> initializing a tulip setup for project %s and Lua %s

Press ENTER to start.
]], base, luaver)
  io.read('l')
  if not opts.yes then
    io.read('l')
  end

  log('> creating LuaRocks project configuration...')
  utils.log('> creating LuaRocks project configuration...')
  create_luarocks(dir, luaver)
  log(' ok\n')
  utils.log(' ok\n')

  log('> creating postgresql database and docker files...')
  create_postgres(dir)
  log(' ok\n')
  utils.log('> creating postgresql database and docker files...')
  create_postgres(dir, opts.port)
  utils.log(' ok\n')

  log('> creating passwords and secrets...')
  create_secrets(dir)
  log(' ok\n')
  utils.log('> creating passwords and secrets...')
  create_secrets(dir, opts.port)
  utils.log(' ok\n')

  if sh.cmd('command', '-v', 'mkcert'):output() then
    log('> creating localhost certificates...\n')
    utils.log('> creating localhost certificates...\n')
    create_certs(dir)
  else
    log('> skipping localhost certificates, mkcert not installed\n')
    utils.log('> skipping localhost certificates, mkcert not installed\n')
  end

  log('> creating direnv envrc file...')
  create_envrc(dir)
  log(' ok\n')
  utils.log('> creating direnv envrc file...')
  create_envrc(dir, opts.port)
  utils.log(' ok\n')

  log('> creating gitignore file...')
  utils.log('> creating gitignore file...')
  create_gitignore(dir)
  log(' ok\n')
  utils.log(' ok\n')

  log('> installing the Tulip LuaRock package and its dependencies...\n')
  utils.log('> installing the Tulip LuaRock package and its dependencies...\n')
  install_tulip(dir)

  log([[
  utils.log([[
> initialization of the tulip setup for project %s and Lua %s completed:

Execute the following

M src/main.lua => src/main.lua +30 -2
@@ 7,18 7,36 @@ Usage: tulip-cli CMD [<options>]

The following tulip commands are supported:

  init                  Initialize environment for development of a
                        tulip project.
  $ tulip-cli init DIR
      Initialize environment for development of a tulip project in DIR,
      which is a path relative to the current working directory.

  $ tulip-cli db reset
      Drops and recreates the database, so that no migration has been
      applied yet. All data will be lost, use with care.

  $ tulip-cli db clean PREFIX
      Cleans the database by removing any databases and users that start
      with PREFIX. Useful if database copies are created for tests.

Options:

  -h, --help            Display this help and exit.
  --port=PORT           Expose the database on that port for the host
                        (default: 5432). Only used for the 'init' command.
  --prefix=PREFIX       Clean databases and users that have this prefix.
                        Only used for the 'db clean' command.
  -V, --version         Display the version and exit.
  -y, --yes             Skip confirmation of actions.
]], VERSION)

local cmds = require 'src.cmds'
local parser = OptionParser(help)

local defaults = {
  port = 5432,
}

return function(args)
  local arg, opts = parser:parse(args)
  if #arg == 0 then


@@ 33,6 51,16 @@ return function(args)
    return
  end

  -- set the default values for options
  if opts.port then
    opts.port = math.tointeger(opts.port)
    if not opts.port then
      parser:opterr('the --port option must be a number')
      return
    end
  end
  opts.port = opts.port or defaults.port

  return f(arg, opts, function(msg, ...)
    parser:opterr(string.format(msg, ...))
  end)

A src/utils.lua => src/utils.lua +27 -0
@@ 0,0 1,27 @@
local base64 = require 'base64'

local rand = assert(io.open('/dev/urandom', 'r'))
local enc = base64.makeencoder('.', '_', '-')

local M = {}

function M.random_string(len)
  local raw = rand:read(len)
  return base64.encode(raw, enc)
end

function M.log(s, ...)
  local msg = string.format(s, ...)
  io.write(msg)
  if not string.match(msg, '\n$') then
    io.flush()
  end
end

function M.write_file(path, content)
  local fd = assert(io.open(path, 'w+'))
  assert(fd:write(content))
  assert(fd:close())
end

return M