~damien/infrastructure

b9a7a9052354499f2bba2cb3bf8facc983a75597 — Damien Radtke 22 days ago 58f3107 master
Re-introducing Packer into this
52 files changed, 379 insertions(+), 298 deletions(-)

A ca/.gitignore
A ca/.terraform.lock.hcl
A ca/README.md
A ca/ca-new
A ca/consul-agent-ca.crt
A ca/consul-agent-ca.srl
A ca/inspect-cert
A ca/nomad-agent-ca.crt
A ca/provision-cert
A ca/vault-server-ca.crt
D config/profile.local
R config/README.md => packer/config/README.md
R config/cfssl.json => packer/config/cfssl.json
R config/consul/base.hcl => packer/config/consul.d/base.hcl
R config/consul/watches.hcl => packer/config/consul.d/watches.hcl
R config/nomad/base.hcl => packer/config/nomad.d/base.hcl
A packer/config/profile.local
R config/vault/base.hcl => packer/config/vault.d/base.hcl
R firewall/README.md => packer/firewall/README.md
R firewall/services/consul-wan.xml => packer/firewall/services/consul-wan.xml
R firewall/services/consul.xml => packer/firewall/services/consul.xml
R firewall/services/fabio.xml => packer/firewall/services/fabio.xml
R firewall/services/nomad-dynamic-ports.xml => packer/firewall/services/nomad-dynamic-ports.xml
R firewall/services/nomad.xml => packer/firewall/services/nomad.xml
R firewall/services/vault.xml => packer/firewall/services/vault.xml
R firewall/services/web.xml => packer/firewall/services/web.xml
R firewall/zones/README => packer/firewall/zones/README
R firewall/zones/nomad-clients.xml => packer/firewall/zones/nomad-clients.xml
R firewall/zones/public.xml => packer/firewall/zones/public.xml
M packer/image.pkr.hcl
M packer/scripts/install-hashicorp.sh
R services/README.md => packer/services/README.md
A packer/services/consul-server@.service
R services/consul.service => packer/services/consul.service
R services/nomad.service => packer/services/nomad.service
R services/support/README.md => packer/services/support/README.md
R services/support/install.sh => packer/services/support/install.sh
R services/support/minio.service => packer/services/support/minio.service
R services/support/multirootca.service => packer/services/support/multirootca.service
R services/vault.service => packer/services/vault.service
A scripts/create-app-user.sh
M terraform/.gitignore
M terraform/cluster/consul-server/main.tf
M terraform/cluster/consul-server/variables.tf
M terraform/cluster/main.tf
M terraform/cluster/nomad-client/main.tf
M terraform/cluster/nomad-server/main.tf
M terraform/cluster/outputs.tf
M terraform/cluster/vault-server/main.tf
D terraform/domains.tf
M terraform/main.tf
D terraform/outputs.tf
A ca/.gitignore => ca/.gitignore +2 -0
@@ 0,0 1,2 @@
*.csr
*.key

A ca/.terraform.lock.hcl => ca/.terraform.lock.hcl +23 -0
@@ 0,0 1,23 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.

provider "registry.terraform.io/linode/linode" {
  version     = "1.19.1"
  constraints = ">= 0.0.1"
  hashes = [
    "h1:UNXJmnbYmZuwMl1q8O/ix2NcyvEI29cAg91ZOjb1qXU=",
    "zh:1b9f22c1882b1451492341ef28d6776f606a244e8a58340c21dd6949015a229d",
    "zh:2557f3415f85c9570e44fa7f147269d12c1e4bf32b5b21b09a347dcf574967de",
    "zh:414eac8b55c4db6e4c44702ee3b99e4912bc59c213c4c881309b7c7434977644",
    "zh:436005c12b777a54f0d713a345f30e831e9110a08a464a0178cf64438ad95b08",
    "zh:441358a02bc23f16830c0665a51508edf46a52cf348b69adbd8b6614fe8d4506",
    "zh:487249165132fe777071dfffaae04f14087c966f67e813d01c770b8cf628ed00",
    "zh:510c500b8dd0639b1e54297664d183498cae20d3ebf7433c258522b462dd17e0",
    "zh:57e80c659478185d0adc03764c28cb950e2ae1ca488b296a8a5d6a661a1b55c5",
    "zh:8a487493cdc40081842c5d95cf187fb1e4ce3f8e0ed17b059ecf536418809cd4",
    "zh:afa4b850893aad1e1e90c46f44fdc21cc906ad2417b8d1575bd94a12b9e25be8",
    "zh:cdaed1da0654cb3d6d644af0b7098adcb361c7491ceaa831e322575fa1cbd790",
    "zh:e1d4a03ed421d4e05c8e4a5c5d15f45e1bdd0e2f51c9dba0dc3d051996684a31",
    "zh:f5c4673c6968d6b58f4530e623c4dd7240491ce6acfef57d074177f1511ea76f",
  ]
}

A ca/README.md => ca/README.md +14 -0
@@ 0,0 1,14 @@
This repo demonstrates a simple approach for provisioning servers with
certificates signed by a custom certificate authority.

In order to test it, you need to define a `linode_token` variable inside
`secrets.tfvars`, and then run:

```sh
$ ./ca-new consul-agent
$ terraform apply -var-file secrets.tfvars
```

This will provision a server with
`/etc/ssl/consul-agent/server1.dc1.consul.{crt,key}`, which represents a
certificate and key that have been signed by the local `consul-agent` CA.

A ca/ca-new => ca/ca-new +21 -0
@@ 0,0 1,21 @@
#!/usr/bin/env bash
#
# Inspired by https://deliciousbrains.com/ssl-certificate-authority-for-local-https-development/
#

set -eu -o pipefail

if [[ $# -ne 1 ]]; then
	echo "usage: $0 <name>"
	exit 1
fi

# Generate a new private key for the CA.
# If you want to output an encrypted private key, which will prompt for a passphrase,
# add a cipher parameter, i.e. -des3
openssl genrsa -out "$1-ca.key" 2048

# Use the key to generate a self-signed certificate.
# See: https://docs.oracle.com/cd/E24191_01/common/tutorials/authz_cert_attributes.html
openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout "$1-ca.key" -out "$1-ca.crt" \
	-subj "/CN=$1 CA"

A ca/consul-agent-ca.crt => ca/consul-agent-ca.crt +19 -0
@@ 0,0 1,19 @@
-----BEGIN CERTIFICATE-----
MIIDFTCCAf2gAwIBAgIUQI6dda+SDRrfeGUTY1Igs1nJ+54wDQYJKoZIhvcNAQEL
BQAwGjEYMBYGA1UEAwwPY29uc3VsLWFnZW50IENBMB4XDTIxMDYyOTIxMDgxNVoX
DTIyMDYyOTIxMDgxNVowGjEYMBYGA1UEAwwPY29uc3VsLWFnZW50IENBMIIBIjAN
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAo8jKPl0tC5055xxoSpxSfo9xFwn5
b8O47zmXJtUHnfljzYr+kh6mYeLRrkDk8qyhZZ8jEQwIMwJua+5OHnhRsv6DSwwv
U4wZjAn5s/vj1VDKa3mIfO2sLLPFshaYvWONxMwoZ5yRVZ55Y7APo3qWBHHccv/D
MVc32QagIiJLY++12VJa+oDKfeysHHZVTmbHosxwJGzc5cA/qUD6Bxjy8b8wV7jn
zhSEiAl+46kXA/TGVEmEdVpILJm4JzT5XNl292mbzW9afXjY3h9K5I04d+qOEBrI
h8HKVO6KTsCnyuZMv+9+sLQuA/w92VGSuV03LuX4VbnKYXyNc6rw4dhjhQIDAQAB
o1MwUTAdBgNVHQ4EFgQURjwfuVDSeUNtjMGPqyZYIsM0uB8wHwYDVR0jBBgwFoAU
RjwfuVDSeUNtjMGPqyZYIsM0uB8wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
AQsFAAOCAQEAdY7Homhv6tedRKDM2b2Bu9gqQODMILuAxymG9BefmnS0IKHfqpHy
y9fHy7pRpTAfmWJSuAC/FHOnEpykN/gFEck5ym75n22YIM3JfwLftV7vhrRmYIqc
wdTGybHtvS0+VxL4S8INQgJgCfutj+b2QZp/S8p8Y9s5zQXMWNYd+oruum3jhsuS
lrtiTB7NRcKtOJyhQ6/XeXIk9VVnmeNJLPQP1t/aPcNkkIBq0zAvWYrOfQ5AENWC
BwGAASpWg+UycKqIHSGAORE7UnK/ESqOp4M6RKCLEjzZ7ELYYoxrwXrqBvV1Q+c0
4swJOtMEWu+lTjz8LRx97BN9j0fwQjk+FQ==
-----END CERTIFICATE-----

A ca/consul-agent-ca.srl => ca/consul-agent-ca.srl +1 -0
@@ 0,0 1,1 @@
2E741F16F5701C92061B3669C7546E4A1AA2C813

A ca/inspect-cert => ca/inspect-cert +13 -0
@@ 0,0 1,13 @@
#!/usr/bin/env bash
#
# From https://www.sslshopper.com/article-most-common-openssl-commands.html
#

set -eu -o pipefail

if [[ $# -ne 1 ]]; then
	echo "usage: $0 <cert>"
	exit 1
fi

openssl x509 -in "$1" -text -noout

A ca/nomad-agent-ca.crt => ca/nomad-agent-ca.crt +19 -0
@@ 0,0 1,19 @@
-----BEGIN CERTIFICATE-----
MIIDEzCCAfugAwIBAgIUXebAuVRMeSR4Xuq5RO2x43cMGu0wDQYJKoZIhvcNAQEL
BQAwGTEXMBUGA1UEAwwObm9tYWQtYWdlbnQgQ0EwHhcNMjEwNjMwMTIyMzE2WhcN
MjIwNjMwMTIyMzE2WjAZMRcwFQYDVQQDDA5ub21hZC1hZ2VudCBDQTCCASIwDQYJ
KoZIhvcNAQEBBQADggEPADCCAQoCggEBALoJG71NMFJEehyiPZsm0j7RqeYWSOd5
bGTN6j/ZT+UFY9Oiq1YjtO8bAnlykQIqQ4U1sXnZkptXvvrTKUjFJABhyzR3bc6N
vFYjjXa5wyw6S0GPD6lcpIJ1KJguKXrzU836+FiGd44e0tXo3m5asEU7OC6Cc3PQ
pvdpY+Hfr//L///vriitud3G3m93Ujo2w660oEtUWbasDksh+dyP12AQs/RP8l0S
usfNTfDd3vcWNuSbDJbaIHg8171ie5jsEggBqhjJJigFcrsw/5mhOyruvptc9i4y
9Zp36wlmhspyzslb8N13UngdUHcKQOvtEkbBwtbdinu2/l6QM/NCRrMCAwEAAaNT
MFEwHQYDVR0OBBYEFPOHlXY/PngXhHzFuqlPtyTmDpX8MB8GA1UdIwQYMBaAFPOH
lXY/PngXhHzFuqlPtyTmDpX8MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
BQADggEBABpz6ml+m9Cyt2zKbA+SpEGSOaRbeJpE7BWnb/oEwLyb09e9CygZZlKa
UMegdXPN8+F9TSnisxa4t9ez+7jZoskApFIGYNFFxr6nuNfif136YvBO9TGXtCFL
lZGOSFJkDIb1d4Q19MJfmbsoX5dV8/xjzy6q/3uR88ySBG8Gr4X792qZtLjj8h2G
7a9SdiCj0zSZ2HiZdcKEiu2W9MBVVBbOLhdpwJ2U2iLWN/ML0Fu+gGS1EtlK76rm
sBOEUI1IUcCaJPO7iAb67cesLlw9fkqDlUY+vJhhx0a1h3e83HwROpMnLFGDRwus
Vm4pEOKLyafXucX6JD9FhDHxm5Ji+NQ=
-----END CERTIFICATE-----

A ca/provision-cert => ca/provision-cert +65 -0
@@ 0,0 1,65 @@
#!/usr/bin/env bash
#
# Example:
#
#   provision-cert \
#       --ca consul-agent \
#       --cn server1.dc2.consul \
#       --addr 1.1.1.1 \
#       --owner consul:consul \
#       --outdir /etc/ssl/consul-agent \
#       --basename consul
#

set -eu -o pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"

while [[ $# -gt 0 ]]; do
	case "$1" in
		--ca) ca="$2"; shift 2;;
		--cn) cn="$2"; shift 2;;
		--addr) addr="$2"; shift 2;;
		--owner) owner="$2"; shift 2;;
		--outdir) outdir="$2"; shift 2;;
		--basename) basename="$2"; shift 2;;
		--days) days="$2"; shift 2;;
		*) echo "unrecognized parameter: $1"; exit 1;;
	esac
done

tmp="$(mktemp --directory)"
trap "rm -rf '${tmp}'" EXIT

# Generate a private key and CSR remotely
ssh root@"${addr}" <<EOF
cd '${outdir}'
openssl req -new -newkey rsa:2048 -nodes -keyout '${basename}.key' -out '${basename}.csr' -subj '/CN=${cn}'
EOF

# Copy the CSR down for cert generation
scp root@"${addr}":"${outdir}/${basename}.csr" "${tmp}/"

# Sign the CSR to get a certificate
if [[ -f "${ca}-ca.srl" ]]; then
	serial_arg="-CAserial ${ca}-ca.srl"
else
	serial_arg="-CAcreateserial"
fi

openssl x509 -req \
	-in "${tmp}/${basename}.csr" \
	-CA "${ca}-ca.crt" \
	-CAkey "${ca}-ca.key" \
	${serial_arg} \
	-out "${tmp}/${basename}.crt" \
	-days "${days:-3650}"
# TODO: add key usages? https://learn.hashicorp.com/tutorials/nomad/security-enable-tls

# Copy the certificate back up
scp "${tmp}/${basename}.crt" root@"${addr}":"${outdir}/${basename}.crt"

# Clean up the remote CSR
ssh root@"${addr}" "rm ${outdir}/${basename}.csr"

# Update the owner.
ssh root@"${addr}" "chown -R ${owner} /etc/ssl/${ca}"

A ca/vault-server-ca.crt => ca/vault-server-ca.crt +19 -0
@@ 0,0 1,19 @@
-----BEGIN CERTIFICATE-----
MIIDFTCCAf2gAwIBAgIUG7nY07/pNKsNw3U57J+LtsLRz/cwDQYJKoZIhvcNAQEL
BQAwGjEYMBYGA1UEAwwPdmF1bHQtc2VydmVyIENBMB4XDTIxMDYzMDEyMjMzMVoX
DTIyMDYzMDEyMjMzMVowGjEYMBYGA1UEAwwPdmF1bHQtc2VydmVyIENBMIIBIjAN
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxdFrL9VNyKYYsnb87egzHbOiw4az
Osq1OQt1qAq+aIlLOjT5XHmcnV/HhZzYqUVZ7CjlsNJBLgRdOwfmj34ABtsy9IP4
cx3aXh/YQ9bH9lxeK7jzXFOarDiIVBtKnDwGR8+ANgpHkps62OimayK7zFec5tvT
jfMaMLnHZ3XrJIFd6uA13r/v/JeUpoizsS7DFbbWxzy4w3aJRCiNPyvLoDvEg54W
MRJflTGbjAX4XUOn4FTkvnWX2cA3zkuk7qEMAO5VMGkrA6IGsDmPSrraQN2Fw4Mf
5NNXoAVfwCJiGMzBiwV/cmxoisFMIvVMNTIHFBg1bIKF1ziIUYWCCC6s5wIDAQAB
o1MwUTAdBgNVHQ4EFgQUrz82qwwUuZG+/FYBwaI0QVcpWewwHwYDVR0jBBgwFoAU
rz82qwwUuZG+/FYBwaI0QVcpWewwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
AQsFAAOCAQEAguEF/gVGimJhqqMifhfP7Oxxmz6v0clsIp/gR4qn9OZ1/F+Xfjkp
cbyfC0urizQYYgfp2ttKV0Q0kzB2Hx+5eGuZQ9UERVDhiCNlQIHnI5ZG3cJD+Xly
TQq+wokOWw7T1ojeqBFUGED5Fc3R9OfF66srfvOmUx5dGHAJk8dJEL6DnJjZ7abm
gYxAgaS4WoWOPlbMl4KFr65Q8VEO/5LnlSu4vDWUj+N4IoD88J7933v7Uhc09sM/
Ik2e/+0mq3+PwDn5bsPzVPFnZxdKTJJktef0wBlSpHvD9/iGhIVguaKanRgQY/Ps
ZjDbq8a5Vp2p1gPZ3ctNBJlgvv0JfDcxtA==
-----END CERTIFICATE-----

D config/profile.local => config/profile.local +0 -11
@@ 1,11 0,0 @@
export CONSUL_HTTP_ADDR=unix:///var/run/consul/consul_https.sock

export NOMAD_ADDR=https://localhost:4646
export NOMAD_CACERT=/etc/ssl/nomad/ca.pem
export NOMAD_CLIENT_CERT=/etc/ssl/nomad/cli.pem
export NOMAD_CLIENT_KEY=/etc/ssl/nomad/cli-key.pem

export VAULT_ADDR=https://localhost:8200
export VAULT_CACERT=/etc/ssl/vault/ca.pem
export VAULT_CLIENT_CERT=/etc/ssl/vault/cli.pem
export VAULT_CLIENT_KEY=/etc/ssl/vault/cli-key.pem

R config/README.md => packer/config/README.md +0 -0
R config/cfssl.json => packer/config/cfssl.json +0 -0
R config/consul/base.hcl => packer/config/consul.d/base.hcl +3 -3
@@ 5,9 5,9 @@ ui = true
# The second (and last) one seems to be the correct one, so we need to offset into the list to get it.
advertise_addr = "{{ GetPublicInterfaces | include `type` `IPv6` | offset -1 | attr `address` }}"

ca_file = "/etc/ssl/consul/ca.pem"
cert_file = "/etc/ssl/consul/consul.pem"
key_file = "/etc/ssl/consul/consul-key.pem"
ca_file = "/etc/ssl/consul-agent/ca.crt"
cert_file = "/etc/ssl/consul-agent/consul.crt"
key_file = "/etc/ssl/consul-agent/consul.key"

addresses {
	https = "[::] unix:///var/run/consul/consul_https.sock"

R config/consul/watches.hcl => packer/config/consul.d/watches.hcl +0 -0
R config/nomad/base.hcl => packer/config/nomad.d/base.hcl +9 -9
@@ 10,18 10,18 @@ advertise {

consul {
        address = "unix:///var/run/consul/consul_https.sock"
        ca_file = "/etc/ssl/consul/ca.pem"
        cert_file = "/etc/ssl/nomad/consul.pem"
        key_file = "/etc/ssl/nomad/consul-key.pem"
        ca_file = "/etc/ssl/consul-agent/ca.crt"
        cert_file = "/etc/ssl/nomad-agent/consul.crt"
        key_file = "/etc/ssl/nomad-agent/consul.key"
}

tls {
        http = true
        rpc = true

        ca_file   = "/etc/ssl/nomad/ca.pem"
        cert_file = "/etc/ssl/nomad/nomad.pem"
        key_file  = "/etc/ssl/nomad/nomad-key.pem"
        ca_file   = "/etc/ssl/nomad-agent/ca.crt"
        cert_file = "/etc/ssl/nomad-agent/nomad.crt"
        key_file  = "/etc/ssl/nomad-agent/nomad.key"

        verify_server_hostname = true
        verify_https_client    = true


@@ 31,9 31,9 @@ vault {
	enabled = true
        address = "https://vault.service.consul:8200"

        ca_file = "/etc/ssl/vault/ca.pem"
        cert_file = "/etc/ssl/nomad/vault.pem"
        key_file = "/etc/ssl/nomad/vault-key.pem"
        ca_file = "/etc/ssl/vault-server/ca.crt"
        cert_file = "/etc/ssl/nomad-agent/vault.crt"
        key_file = "/etc/ssl/nomad-agent/vault.key"

	// The Vault token is set in a systemd override file defined at provision time.
}

A packer/config/profile.local => packer/config/profile.local +12 -0
@@ 0,0 1,12 @@
export CONSUL_HTTP_ADDR=unix:///var/run/consul/consul_https.sock
export CONSUL_CACERT=/etc/ssl/consul-agent/ca.crt

export NOMAD_ADDR=https://localhost:4646
export NOMAD_CACERT=/etc/ssl/nomad-agent/ca.crt
export NOMAD_CLIENT_CERT=/etc/ssl/nomad-agent/cli.crt
export NOMAD_CLIENT_KEY=/etc/ssl/nomad-agent/cli.key

export VAULT_ADDR=https://localhost:8200
export VAULT_CACERT=/etc/ssl/vault-server/ca.crt
export VAULT_CLIENT_CERT=/etc/ssl/vault-server/cli.crt
export VAULT_CLIENT_KEY=/etc/ssl/vault-server/cli.key

R config/vault/base.hcl => packer/config/vault.d/base.hcl +2 -2
@@ 8,8 8,8 @@ storage "consul" {
listener "tcp" {
        address = "[::]:8200"

        tls_cert_file = "/etc/ssl/vault/vault.pem"
        tls_key_file = "/etc/ssl/vault/vault-key.pem"
        tls_cert_file = "/etc/ssl/vault-server/vault.crt"
        tls_key_file = "/etc/ssl/vault-server/vault.key"
}

api_addr = "https://vault.service.consul:8200"

R firewall/README.md => packer/firewall/README.md +0 -0
R firewall/services/consul-wan.xml => packer/firewall/services/consul-wan.xml +0 -0
R firewall/services/consul.xml => packer/firewall/services/consul.xml +0 -0
R firewall/services/fabio.xml => packer/firewall/services/fabio.xml +0 -0
R firewall/services/nomad-dynamic-ports.xml => packer/firewall/services/nomad-dynamic-ports.xml +0 -0
R firewall/services/nomad.xml => packer/firewall/services/nomad.xml +0 -0
R firewall/services/vault.xml => packer/firewall/services/vault.xml +0 -0
R firewall/services/web.xml => packer/firewall/services/web.xml +0 -0
R firewall/zones/README => packer/firewall/zones/README +0 -0
R firewall/zones/nomad-clients.xml => packer/firewall/zones/nomad-clients.xml +0 -0
R firewall/zones/public.xml => packer/firewall/zones/public.xml +0 -0
M packer/image.pkr.hcl => packer/image.pkr.hcl +39 -25
@@ 1,3 1,4 @@
// TODO: move resources required by packer into this folder?
packer {
  required_plugins {
    linode = {


@@ 65,31 66,29 @@ build {
  provisioner "shell" {
    inline = [
      "zypper --non-interactive install wget jq firewalld moreutils",
      "zypper --non-interactive clean",
      "echo Updating CA certificates",
      "update-ca-certificates --verbose",
      "mkdir /etc/consul.d",
      "mkdir /etc/nomad.d",
      "mkdir /etc/vault.d",
      "echo Making directories",
      "mkdir /etc/ssl/consul-agent",
      "mkdir /etc/ssl/nomad-agent",
      "mkdir /etc/ssl/vault-server",
    ]
  }

  provisioner "file" {
  	destination = "/etc/consul.d/base.hcl"
    source      = "config/consul/base.hcl"
  	destination = "/etc"
    source      = "config/consul.d"
  }

  provisioner "file" {
    destination = "/etc/nomad.d/base.hcl"
    source      = "config/nomad/base.hcl"
    destination = "/etc"
    source      = "config/nomad.d"
  }

  provisioner "file" {
    destination = "/etc/vault.d/vault.hcl"
    source      = "config/vault/base.hcl"
  }

  provisioner "file" {
    destination = "/usr/local/bin"
    source      = "scripts/"
    destination = "/etc"
    source      = "config/vault.d"
  }

  provisioner "file" {


@@ 108,39 107,46 @@ build {
  }

  provisioner "shell" {
    script = "packer/scripts/install-hashicorp.sh"
    script = "scripts/install-hashicorp.sh"
    environment_vars = ["APP_NAME=consul", "APP_VERSION=${var.consul_version}"]
  }

  provisioner "shell" {
    script = "packer/scripts/install-hashicorp.sh"
    script = "scripts/install-hashicorp.sh"
    environment_vars = ["APP_NAME=nomad", "APP_VERSION=${var.nomad_version}"]
  }

  provisioner "shell" {
    script = "packer/scripts/install-hashicorp.sh"
    script = "scripts/install-hashicorp.sh"
    environment_vars = ["APP_NAME=vault", "APP_VERSION=${var.vault_version}"]
  }

  provisioner "file" {
    destination = "/etc/ssl/consul/ca.pem"
    source      = "certs/consul-ca.pem"
    destination = "/etc/ssl/consul-agent/ca.crt"
    source      = "../ca/consul-agent-ca.crt"
  }

  provisioner "file" {
    destination = "/etc/ssl/nomad/ca.pem"
    source      = "certs/nomad-ca.pem"
    destination = "/etc/ssl/nomad-agent/ca.crt"
    source      = "../ca/nomad-agent-ca.crt"
  }

  provisioner "file" {
    destination = "/etc/ssl/cfssl.json"
    source      = "config/cfssl.json"
    destination = "/etc/ssl/vault-server/ca.crt"
    source      = "../ca/vault-server-ca.crt"
  }

  /*
  provisioner "file" {
    destination = "/etc/ssl/nomad/ca.pem"
    source      = "certs/nomad-ca.pem"
  }

  provisioner "file" {
    destination = "/etc/ssl/vault/ca.pem"
    source      = "certs/vault-ca.pem"
  }
  */

  provisioner "file" {
    destination = "/etc/profile.local"


@@ 149,8 155,16 @@ build {

  provisioner "shell" {
    inline = [
      "wget --quiet -O /usr/local/bin/cfssl https://github.com/cloudflare/cfssl/releases/download/v${var.cfssl_version}/cfssl_${var.cfssl_version}_linux_amd64",
      "chmod +x /usr/local/bin/cfssl",
      "mkdir /home/damien",
      "mkdir /home/damien/.ssh",
      "touch /home/damien/.bashrc",
      "useradd --home-dir /home/damien damien",
      "chown -R damien:users /home/damien",
      "echo 'damien ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/damien",
      "sudo -u damien /usr/local/bin/consul -autocomplete-install",
      "sudo -u damien /usr/local/bin/nomad -autocomplete-install",
      "sudo -u damien /usr/local/bin/vault -autocomplete-install",
      "usermod --append --groups consul damien",
    ]
  }
}

M packer/scripts/install-hashicorp.sh => packer/scripts/install-hashicorp.sh +1 -3
@@ 160,18 160,16 @@ pushd /tmp
popd

home_dir="/var/lib/${APP_NAME}"
certs_dir="/etc/ssl/${APP_NAME}"

# This function copies client configs into /etc/<app>.d,
# creates a user and group named after the app, creates
# a home directory for them, and then enables+starts it.
echo "[Setting up ${APP_NAME}]"
mkdir -p "${home_dir}" "${certs_dir}"
mkdir -p "${home_dir}"
groupadd "${APP_NAME}"
useradd --home-dir "${home_dir}" --gid "${APP_NAME}" "${APP_NAME}"

chown -R "${APP_NAME}:${APP_NAME}" "${home_dir}"
chown -R "${APP_NAME}:${APP_NAME}" "${certs_dir}"

usermod -a -G "${APP_NAME}" damien


R services/README.md => packer/services/README.md +0 -0
A packer/services/consul-server@.service => packer/services/consul-server@.service +19 -0
@@ 0,0 1,19 @@
[Unit]
Description="Consul Server"
Documentation=https://www.consul.io/
Requires=network-online.target
After=network-online.target

[Service]
User=consul
Group=consul
ExecStartPre=+/usr/local/bin/create-app-user.sh consul "consul%i"
ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/
ExecReload=/usr/local/bin/consul reload -ca-file=/etc/ssl/consul-agent/ca.crt -http-addr=unix:///var/run/consul/consul_https.sock
KillMode=process
Restart=on-failure
LimitNOFILE=65536
WorkingDirectory=/var/lib/consul

[Install]
WantedBy=multi-user.target

R services/consul.service => packer/services/consul.service +1 -1
@@ 10,7 10,7 @@ Group=consul
ExecStartPre=+/usr/bin/mkdir -p /var/run/consul
ExecStartPre=+/usr/bin/chown -R consul:consul /var/run/consul
ExecStart=/usr/local/bin/consul agent -config-dir=/etc/consul.d/
ExecReload=/usr/local/bin/consul reload -ca-file=/etc/ssl/consul/ca.pem -http-addr=unix:///var/run/consul/consul_https.sock
ExecReload=/usr/local/bin/consul reload -ca-file=/etc/ssl/consul-agent/ca.crt -http-addr=unix:///var/run/consul/consul_https.sock
KillMode=process
Restart=on-failure
LimitNOFILE=65536

R services/nomad.service => packer/services/nomad.service +0 -0
R services/support/README.md => packer/services/support/README.md +0 -0
R services/support/install.sh => packer/services/support/install.sh +0 -0
R services/support/minio.service => packer/services/support/minio.service +0 -0
R services/support/multirootca.service => packer/services/support/multirootca.service +0 -0
R services/vault.service => packer/services/vault.service +0 -0
A scripts/create-app-user.sh => scripts/create-app-user.sh +13 -0
@@ 0,0 1,13 @@
#!/usr/bin/env bash

name="$1"
dir="$2"

groupadd --force "${name}"
useradd --gid "${name}" --home-dir /var/lib/"${dir}" "${name}"

mkdir -p /var/lib/"${dir}"
mkdir -p /var/run/"${dir}"

chown -R "${name}":"${name}" /var/lib/"${dir}"
chown -R "${name}":"${name}" /var/run/"${dir}"

M terraform/.gitignore => terraform/.gitignore +1 -0
@@ 2,3 2,4 @@
*.tfstate
*.tfstate.*
debug.log
.terraform.lock.hcl

M terraform/cluster/consul-server/main.tf => terraform/cluster/consul-server/main.tf +51 -102
@@ 1,142 1,76 @@
terraform {
  required_providers {
    linode = {
      source  = "linode/linode"
      version = "1.19.1"
    }
  }
}

locals {
  addrs = [for ipv6 in linode_instance.servers.*.ipv6 : split("/", ipv6)[0]]
}

resource "linode_instance" "servers" {
  count            = var.servers
  label            = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  label            = "consul-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type
  authorized_users = var.authorized_users
  group            = terraform.workspace

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    consul_version = random_id.servers[count.index].keepers.consul_version
  }

  lifecycle {
    create_before_destroy = true
  }

  // wait for stackscript to complete
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "while ! [[ -f /root/StackScript.complete ]]; do echo 'waiting for stackscript to complete...'; sleep 3; done",
    ]
    inline = ["echo SSH is up"]
  }

  // systemd service files
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/systemd/system/"
    source      = "../services/"
  }

  // cfssl config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/cfssl.json"
    content     = data.template_file.cfssl_config.rendered
  // Recognize the new server
  // This isn't _ideal_, but it's better than disabling host key checking for every SSH command.
  provisioner "local-exec" {
    command = "ssh-keygen -R '${self.ip_address}' && ssh-keyscan -v '${self.ip_address}' >> ~/.ssh/known_hosts"
  }

  // consul config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d"
    source      = "../config/consul"
  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca consul-agent --cn server.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
  }

  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d/server.hcl"
    content     = <<-EOT
      node_name        = "${self.label}"
      datacenter       = "${var.datacenter}"
      server           = true
      bootstrap_expect = ${var.servers}
    EOT
  }

  // firewall services
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/firewalld/services"
    source      = "../firewall/services/"
  }

  // firewall zones
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/firewalld/zones"
    source      = "../firewall/zones/"
  }

  // issue-cert script
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/usr/local/bin/issue-cert.sh"
    source      = "../scripts/issue-cert.sh"
  }

  // Consul certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/consul/ca.pem"
    source      = "/etc/ssl/consul/ca.pem"
  }

  // set global environment variables
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/profile.local"
    content     = <<-EOT
      export CONSUL_HTTP_ADDR=unix:///var/run/consul/consul_https.sock
    EOT
  }

  // reload firewall
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["service firewalld reload"]
  }

  // fix permissions
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "chown -R consul:consul /etc/consul.d",
      "chmod +x /usr/local/bin/issue-cert.sh",
      "chmod 0400 /etc/ssl/cfssl.json",
    ]
  }

  // issue certs
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul --hostnames ${split("/", self.ipv6)[0]}",
    ]
  }

  // start services
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["systemctl enable consul && service consul start"]
  }

  // install autocompletion
  // set hostname
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "sudo -u damien /usr/local/bin/consul -autocomplete-install",
    ]
    inline = ["hostnamectl set-hostname '${self.label}'"]
  }

  // disable further root ssh
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "sed -i 's/PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config",
      "service sshd reload",
      "cp /root/.ssh/authorized_keys /home/damien/.ssh/",
      "chown -R damien:users /home/damien/.ssh/",
      // TODO: re-disable root, but only after null resource provisioners have also run
      //"sed -i 's/PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config",
      //"service sshd reload",
    ]
  }
}


@@ 147,16 81,31 @@ resource "random_id" "servers" {
    datacenter     = var.datacenter
    image          = var.image
    instance_type  = var.instance_type
    consul_version = var.consul_version
  }
  byte_length = 3
  byte_length = 4
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key
// The join.hcl file is provisioned via a null resource because attempting to put
// that logic in a provisioner on the instance directly results in a cycle error.
resource "null_resource" "cluster" {
  triggers = {
    ips = "${join(",", linode_instance.servers.*.ipv6)}"
  }

  count = length(linode_instance.servers.*.id)
  connection {
    host = split("/", element(linode_instance.servers.*.ipv6, count.index))[0]
  }

  provisioner "file" {
    destination = "/etc/consul.d/join.hcl"
    content = <<EOF
retry_join = [${join(", ", [for ip in linode_instance.servers.*.ipv6 : format("\"%s\"", split("/", ip)[0]) if ip != linode_instance.servers[count.index].ipv6])}]
EOF
  }

  provisioner "remote-exec" {
    inline = ["service consul restart"]
  }
}


M terraform/cluster/consul-server/variables.tf => terraform/cluster/consul-server/variables.tf +0 -6
@@ 3,12 3,6 @@ variable servers { type = number }
variable datacenter { type = string }
variable image { type = string }
variable instance_type { type = string }
variable stackscript_id { type = number }
variable authorized_users { type = list(string) }

variable ca_host { type = string }
variable ca_key { type = string }

variable consul_version { type = string }

// vim: set expandtab shiftwidth=2 tabstop=2:

M terraform/cluster/main.tf => terraform/cluster/main.tf +6 -30
@@ 1,36 1,23 @@
variable linode_token { type = string }
variable ca_host { type = string }
variable ca_key { type = string }
// variable linode_token { type = string }
variable vault_token { type = string }

variable region { type = string }
variable datacenter { type = string }
variable image { type = string }
variable instance_type { type = string }
variable authorized_users { type = list(string) }

variable consul_version { type = string }
variable nomad_version { type = string }
variable vault_version { type = string }

locals {
  stackscript_id   = 535217
}

module "consul-server" {
  source = "./consul-server"

  servers        = 1
  consul_version = var.consul_version
  servers        = 3

  datacenter       = var.region
  datacenter       = var.datacenter
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

/*
module "nomad-server" {
  source = "./nomad-server"



@@ 42,10 29,7 @@ module "nomad-server" {
  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
  vault_token      = var.vault_token
}



@@ 60,10 44,7 @@ module "nomad-client" {
  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "nomad-client-load-balancer" {


@@ 78,10 59,7 @@ module "nomad-client-load-balancer" {
  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "vault-server" {


@@ 95,8 73,6 @@ module "vault-server" {
  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}
*/

M terraform/cluster/nomad-client/main.tf => terraform/cluster/nomad-client/main.tf +2 -2
@@ 16,7 16,7 @@ locals {

resource "linode_instance" "clients" {
  count  = var.clients
  label  = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
  label  = "nomadclient-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
  region = random_id.clients[count.index].keepers.datacenter
  image  = random_id.clients[count.index].keepers.image
  type   = random_id.clients[count.index].keepers.instance_type


@@ 26,7 26,7 @@ resource "linode_instance" "clients" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
    hostname       = "nomadclient-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
    consul_version = random_id.clients[count.index].keepers.consul_version
    nomad_version  = random_id.clients[count.index].keepers.nomad_version
  }

M terraform/cluster/nomad-server/main.tf => terraform/cluster/nomad-server/main.tf +2 -2
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  label            = "nomadserver-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 9,7 9,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    hostname       = "nomadserver-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    nomad_version  = random_id.servers[count.index].keepers.nomad_version
  }

M terraform/cluster/outputs.tf => terraform/cluster/outputs.tf +2 -0
@@ 3,6 3,7 @@ output "consul-servers" {
  value       = module.consul-server.instances
}

/*
output "nomad-servers" {
  description = "Nomad server instances"
  value       = module.nomad-server.instances


@@ 22,3 23,4 @@ output "vault-servers" {
  description = "Vault server instances"
  value       = module.vault-server.instances
}
*/

M terraform/cluster/vault-server/main.tf => terraform/cluster/vault-server/main.tf +3 -3
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  label            = "vault-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 9,7 9,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    hostname       = "vault-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    vault_version  = random_id.servers[count.index].keepers.vault_version
  }


@@ 202,7 202,7 @@ resource "random_id" "servers" {
    consul_version = var.consul_version
    vault_version  = var.vault_version
  }
  byte_length = 3
  byte_length = 4
}

data "template_file" "cfssl_config" {

D terraform/domains.tf => terraform/domains.tf +0 -42
@@ 1,42 0,0 @@
module "damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  instances = module.cluster-blue.nomad-client-load-balancers
}

module "www-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "www"
  instances = module.cluster-blue.nomad-client-load-balancers
}

module "photos-radtke-family" {
  source    = "./domain-address"
  domain    = "radtke.family"
  name      = "photos"
  instances = module.cluster-blue.nomad-client-load-balancers
}

module "consul-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "consul"
  instances = [module.cluster-blue.consul-servers[0]]
}

module "nomad-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "nomad"
  instances = [module.cluster-blue.nomad-servers[0]]
}

/*
module "vault-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "vault"
  instances = flatten([[module.vault-server-blue.instances[0]], [module.vault-server-green.instances[0]]])
}
*/

M terraform/main.tf => terraform/main.tf +17 -24
@@ 1,32 1,25 @@
terraform {
  required_providers {
    linode = {
      source  = "linode/linode"
      version = "1.19.1"
    }
  }
}

variable "linode_token" { type = string }

provider "linode" {
  token = var.linode_token
}

provider "random" {}

data "linode_profile" "me" {}

variable linode_token { type = string }
variable ca_host { type = string }
variable ca_key { type = string }
variable vault_token { type = string }

module "cluster-blue" {
	source = "./cluster"

	linode_token = var.linode_token
	ca_host = var.ca_host
	ca_key = var.ca_key
	vault_token = var.vault_token

	region = "ca-central"
	image = "linode/opensuse15.2"
  instance_type    = "g6-nanode-1"
module "cluster" {
  source           = "./cluster"
  datacenter       = "ca-central"
  image            = "private/12878715"
  authorized_users = [data.linode_profile.me.username]

  consul_version = "1.9.0"
  nomad_version  = "1.0.1"
  vault_version  = "1.6.1"
  instance_type    = "g6-nanode-1"
  vault_token      = "root_token"
}

// TODO: cluster-green

D terraform/outputs.tf => terraform/outputs.tf +0 -33
@@ 1,33 0,0 @@
// TODO: standardize output names?

output "consul_server_ips" {
  value = compact(flatten([
		[for ip in module.cluster-blue.consul-servers[*].ipv6 : split("/", ip)[0]],
	]))
}

output "nomad_server_ips" {
  value = compact(flatten([
		[for ip in module.cluster-blue.nomad-servers[*].ipv6 : split("/", ip)[0]],
	]))
}

output "nomad_client_ips" {
  /*
	value = concat(
		[for ip in module.nomad-client.instances[*].ipv6: split("/", ip)[0]],
		[for ip in module.nomad-client-load-balancer.instances[*].ipv6: split("/", ip)[0]],
	)
	*/
  value = flatten(concat(
    module.cluster-blue.nomad-clients[*].ip_address,
    module.cluster-blue.nomad-client-load-balancers[*].ip_address,
  ))
}

output "vault_server_ips" {
  value = compact(flatten([
		[for ip in module.cluster-blue.vault-servers[*].ipv6 : split("/", ip)[0]],
	]))
}