~damien/infrastructure

29857ed6e008cbc1af7183276345a62947e8a9ac — Damien Radtke 10 months ago 3bb3153
More random updates
M config/nomad/base.hcl => config/nomad/base.hcl +3 -1
@@ 28,10 28,12 @@ tls {
}

vault {
        enabled = false
	enabled = true
        address = "https://vault.service.consul:8200"

        ca_file = "/etc/ssl/vault/ca.pem"
        cert_file = "/etc/ssl/nomad/vault.pem"
        key_file = "/etc/ssl/nomad/vault-key.pem"

	// The Vault token is set in a systemd override file defined at provision time.
}

D config/nomad/client.hcl.erb => config/nomad/client.hcl.erb +0 -29
@@ 1,29 0,0 @@
advertise {
        http = "{{GetInterfaceIP `eth0`}}"
        rpc = "{{GetInterfaceIP `eth0`}}"
        serf = "{{GetInterfaceIP `eth0`}}"
}

client {
        enabled = true

        # This was borrowed from https://www.nomadproject.io/docs/drivers/exec.html#chroot,
        # minus /lib32 since it doesn't exist on openSUSE and plus the SSL certs directory.
        # The certs are needed to be able to make HTTPS requests.
        chroot_env = {
                "/bin" = "/bin"
                "/etc" = "/etc"
                "/lib" = "/lib"
                "/lib64" = "/lib64"
                "/run/resolvconf" = "/run/resolvconf"
                "/sbin" = "/sbin"
                "/usr" = "/usr"
                "/etc/ssl/certs" = "/etc/ssl/certs"
        }
}

plugin "raw_exec" {
        config {
                enabled = true
        }
}

D config/nomad/server.hcl.erb => config/nomad/server.hcl.erb +0 -11
@@ 1,11 0,0 @@
server {
	enabled = true
        bootstrap_expect = 1
}

#vault {
#	# TODO: the docs recommend not defining this here in favor of having VAULT_TOKEN
#	# defined when starting the service. Might be worth considering, but as long
#	# as this isn't checked into VCS with a real token it should be fine.
#	token = "..."
#}

M jobs/README.md => jobs/README.md +4 -4
@@ 5,17 5,17 @@ This folder contains Nomad job definitions.
First, make sure your `.bashrc` is set up correctly with something like

```bash
export NOMAD_ADDR="https://[2600:3c04::f03c:92ff:fe3e:6fbe]:4646"
export NOMAD_ADDR="https://[2600:3c04::f03c:92ff:fed4:f455]:4646"
export NOMAD_CACERT="/etc/ssl/nomad/ca.pem"
export NOMAD_CLIENT_CERT="${HOME}/nomad-cli.pem"
export NOMAD_CLIENT_KEY="${HOME}/nomad-cli-key.pem"
```

Then you can submit jobs with the `nomad` command, or use `compile.sh` if it
requires substiution of secret values:
Then you can submit jobs with the `nomad` command, or use `nomad-compile` from
the tools directory if it requires substiution of secret values:

```bash
$ ./compile.sh job.nomad | nomad job run -
$ nomad-compile job.nomad.erb | nomad job run -
```

## Running (old way)

D jobs/acme-renewer.nomad => jobs/acme-renewer.nomad +0 -212
@@ 1,212 0,0 @@
job "acme-renewer" {
	region = "us"

	datacenters = ["us-central"]
	type = "batch"

	periodic {
	 	cron = "0 1 * * 0"  // 1am on Sundays
	 	time_zone = "America/Chicago"
	}

	// Disable rescheduling because otherwise it's easy to be rate-limited.
	reschedule {
		attempts = 0
		unlimited = false
	}

	# TODO: the next few groups are all identical save for the domain being renewed.
	# It might be a good idea to look into templating this more to reduce duplication.

	group "damienradtke.com" {
		// Disable automatic restarts because otherwise it's easy to be rate-limited.
		restart {
			attempts = 0
		}

		task "renew" {
			// Network issues using plain exec?
			driver = "raw_exec"
			config {
				command = "renew-https-cert.sh"
				args = ["damienradtke.com", "--issue", "--log"]  # add --force if needed
			}

			user = "nobody"

			artifact {
				source = "https://github.com/Neilpang/acme.sh/archive/2.8.2.tar.gz"
				options {
					checksum = "sha256:9c97ae15db3fc65200db462b3304aa082b1367f1ba4af5a86693b014a991c990"
				}
			}

			artifact {
				source = "s3::http://${MINIO_HOST}:9000/artifacts/renew-https-cert.sh"
				options {
					aws_access_key_id = "${MINIO_ACCESS_KEY}"
					aws_access_key_secret = "${MINIO_SECRET_KEY}"
				}
			}

			resources {
				cpu = 20  // MHz, the minimum value
				memory = 30  // MB
			}

			vault {
				policies = ["acme-renewer"]
			}

			env {
				LINODE_V4_API_KEY = "30a5c8c6cff7283a99c4a9d744d860ec3af6fb8776cf19aa5fd1cd7fa5586acb"
				VAULT_CAPATH = "/etc/ssl/vault/ca.pem"
				VAULT_ADDR = "https://vault.service.consul:8200"
			}
		}
	}

	group "www.damienradtke.com" {
		// Disable automatic restarts because otherwise it's easy to be rate-limited.
		restart {
			attempts = 0
		}

		task "renew" {
			// Network issues using plain exec?
			driver = "raw_exec"
			config {
				command = "renew-https-cert.sh"
				args = ["www.damienradtke.com", "--issue", "--log"]  # add --force if needed
			}

			user = "nobody"

			artifact {
				source = "https://github.com/Neilpang/acme.sh/archive/2.8.2.tar.gz"
				options {
					checksum = "sha256:9c97ae15db3fc65200db462b3304aa082b1367f1ba4af5a86693b014a991c990"
				}
			}

			artifact {
				source = "s3::http://${MINIO_HOST}:9000/artifacts/renew-https-cert.sh"
				options {
					aws_access_key_id = "${MINIO_ACCESS_KEY}"
					aws_access_key_secret = "${MINIO_SECRET_KEY}"
				}
			}

			resources {
				cpu = 20  // MHz, the minimum value
				memory = 30  // MB
			}

			vault {
				policies = ["acme-renewer"]
			}

			env {
				LINODE_V4_API_KEY = "30a5c8c6cff7283a99c4a9d744d860ec3af6fb8776cf19aa5fd1cd7fa5586acb"
				VAULT_CAPATH = "/etc/ssl/vault/ca.pem"
				VAULT_ADDR = "https://vault.service.consul:8200"
			}
		}
	}

	group "mail.damienradtke.com" {
		// Disable automatic restarts because otherwise it's easy to be rate-limited.
		restart {
			attempts = 0
		}

		task "renew" {
			// Network issues using plain exec?
			driver = "raw_exec"
			config {
				command = "renew-https-cert.sh"
				args = ["mail.damienradtke.com", "--issue", "--log"]  # add --force if needed
			}

			user = "nobody"

			artifact {
				source = "https://github.com/Neilpang/acme.sh/archive/2.8.2.tar.gz"
				options {
					checksum = "sha256:9c97ae15db3fc65200db462b3304aa082b1367f1ba4af5a86693b014a991c990"
				}
			}

			artifact {
				source = "s3::http://${MINIO_HOST}:9000/artifacts/renew-https-cert.sh"
				options {
					aws_access_key_id = "${MINIO_ACCESS_KEY}"
					aws_access_key_secret = "${MINIO_SECRET_KEY}"
				}
			}

			resources {
				cpu = 20  // MHz, the minimum value
				memory = 30  // MB
			}

			vault {
				policies = ["acme-renewer"]
			}

			env {
				LINODE_V4_API_KEY = "30a5c8c6cff7283a99c4a9d744d860ec3af6fb8776cf19aa5fd1cd7fa5586acb"
				VAULT_CAPATH = "/etc/ssl/vault/ca.pem"
				VAULT_ADDR = "https://vault.service.consul:8200"
			}
		}
	}

	group "radtke.family" {
		// Disable automatic restarts because otherwise it's easy to be rate-limited.
		restart {
			attempts = 0
		}

		task "renew" {
			// Network issues using plain exec?
			driver = "raw_exec"
			config {
				command = "renew-https-cert.sh"
				args = ["radtke.family", "--issue", "--log"]  # add --force if needed
			}

			user = "nobody"

			artifact {
				source = "https://github.com/Neilpang/acme.sh/archive/2.8.2.tar.gz"
				options {
					checksum = "sha256:9c97ae15db3fc65200db462b3304aa082b1367f1ba4af5a86693b014a991c990"
				}
			}

			artifact {
				source = "s3::http://${MINIO_HOST}:9000/artifacts/renew-https-cert.sh"
				options {
					aws_access_key_id = "${MINIO_ACCESS_KEY}"
					aws_access_key_secret = "${MINIO_SECRET_KEY}"
				}
			}

			resources {
				cpu = 20  // MHz, the minimum value
				memory = 30  // MB
			}

			vault {
				policies = ["acme-renewer"]
			}

			env {
				LINODE_V4_API_KEY = "30a5c8c6cff7283a99c4a9d744d860ec3af6fb8776cf19aa5fd1cd7fa5586acb"
				VAULT_CAPATH = "/etc/ssl/vault/ca.pem"
				VAULT_ADDR = "https://vault.service.consul:8200"
			}
		}
	}
}

A jobs/acme-renewer.nomad.erb => jobs/acme-renewer.nomad.erb +90 -0
@@ 0,0 1,90 @@
job "acme-renewer" {
	region = "global"
	datacenters = ["ca-central"]
	type = "batch"

	periodic {
	 	cron = "0 1 * * 0"  // 1am on Sundays
	 	time_zone = "America/Chicago"
	}

	// Disable rescheduling because otherwise it's easy to be rate-limited.
	reschedule {
		attempts = 0
		unlimited = false
	}

	<%
	  @acme_sh_version = "2.8.5"
	%>

	# TODO: add www.damienradtke.com, radtke.family
	# also add something like this somewhere: 
	# "--reloadcmd", "vault kv put secret/fabio/certs/<domain> cert=@'${NOMAD_SECRETS_DIR}/fullchain.pem' key=@'${NOMAD_SECRETS_DIR}/key.pem'",
	#
	# TODO: for reloadcmd, create a script that can use Vault's API to post the cert and key
	<% ["damienradtke.com"].each do |domain| %>
		group "<%= domain %>" {
			// Disable automatic restarts because otherwise it's easy to be rate-limited.
			restart {
				attempts = 0
			}

			task "renew" {
				// Network issues using plain exec?
				driver = "raw_exec"
				config {
					command = "${NOMAD_TASK_DIR}/acme.sh-<%= @acme_sh_version %>/acme.sh"
					args = [
						"--dns", "dns_linode_v4",
						"--dnssleep", "1800",
						"--cert-file", "${NOMAD_SECRETS_DIR}/cert.pem",
						"--fullchain-file", "${NOMAD_SECRETS_DIR}/fullchain.pem",
						"--key-file", "${NOMAD_SECRETS_DIR}/key.pem",
						"--home", "${NOMAD_TASK_DIR}",
						"--domain", "<%= domain %>",
						"--reloadcmd", "sh '${NOMAD_TASK_DIR}/vault-write-certs.sh' --domain '<%= domain %>' --cert-path '${NOMAD_SECRETS_DIR}/fullchain.pem' --key-path '${NOMAD_SECRETS_DIR}/key.pem'",
						"--issue",
						"--log",
					]
				}

				user = "nobody"

				artifact {
					source = "https://github.com/acmesh-official/acme.sh/archive/<%= @acme_sh_version %>.tar.gz"
					/*
					options {
						checksum = "sha256:45d964de8970096dae06aaa45dba2d9d09a41c0a43355191ee627eb00ba5db45"
					}
					*/
				}

				artifact {
					source = "s3::http://45.33.126.243:9000/artifacts/vault-write-certs.sh"
					options {
						aws_access_key_id = "<%= secret('minio', 'access_key') %>"
						aws_access_key_secret = "<%= secret('minio', 'secret_key') %>"
					}
				}

				resources {
					cpu = 20  // MHz, the minimum value
					memory = 30  // MB
				}

				vault {
					policies = ["acme-renewer"]
				}

				env {
					LINODE_V4_API_KEY = "<%= secret('linode/acme-renewer', 'api_key') %>"
					VAULT_CAPATH = "/etc/ssl/vault/ca.pem"
					VAULT_ADDR = "https://vault.service.consul:8200"
				}
			}
		}
	<% end %>
}

// vim: set tabstop=4 shiftwidth=4:

R jobs/cat-facts.nomad => jobs/cat-facts.nomad.erb +6 -4
@@ 17,15 17,15 @@ job "cat-facts" {
			}

			artifact {
				source = "s3::http://${MINIO_HOST}:9000/artifacts/rusty-cat-fact/target/debug/rusty-cat-fact"
				source = "s3::http://45.33.126.243:9000/artifacts/rusty-cat-fact/target/debug/rusty-cat-fact"
				options {
					aws_access_key_id = "${MINIO_ACCESS_KEY}"
					aws_access_key_secret = "${MINIO_SECRET_KEY}"
					aws_access_key_id = "<%= secret('minio', 'access_key') %>"
					aws_access_key_secret = "<%= secret('minio', 'secret_key') %>"
				}
			}

			env {
				WEBHOOK_URL = "${CATFACTS_WEBHOOK}"
				WEBHOOK_URL = "<%= secret('cat_facts', 'webhook_url') %>"
			}

			resources {


@@ 35,3 35,5 @@ job "cat-facts" {
		}
	}
}

// vim: set tabstop=4 shiftwidth=4:

M jobs/damienradtkecom.nomad => jobs/damienradtkecom.nomad +6 -4
@@ 1,7 1,7 @@
job "damienradtkecom" {
	region = "us"
	region = "global"

	datacenters = ["us-central"]
	datacenters = ["ca-central"]
	type = "service"

	group "server" {


@@ 58,11 58,13 @@ job "damienradtkecom" {
			}

			artifact {
				source = "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_Linux-64bit.tar.gz"
				source = "https://github.com/gohugoio/hugo/releases/download/v0.69.2/hugo_0.69.2_Linux-64bit.tar.gz"
				options {
					checksum = "sha256:b92c47a705ad372887454644f8bee76caa6234be13c073834827b58f73fb7adb"
					checksum = "sha256:167d8fb2db1728c0b24914030bab498e38d8ae1dcb01f792672b8a0085903ddf"
				}
			}
		}
	}
}

// vim: set tabstop=4 shiftwidth=4:

M jobs/fabio.nomad => jobs/fabio.nomad +7 -2
@@ 8,7 8,7 @@ job "fabio" {
		task "fabio" {
			driver = "exec"
			config {
				command = "fabio-1.5.11-go1.11.5-linux_amd64",
				command = "fabio-1.5.13-go1.13.4-linux_amd64",
				args = [
					"-proxy.cs", "cs=mycerts;type=vault;cert=secret/fabio/certs",
					"-proxy.addr", "0.0.0.0:9999;cs=mycerts",


@@ 16,7 16,10 @@ job "fabio" {
			}

			artifact {
				source = "https://github.com/fabiolb/fabio/releases/download/v1.5.11/fabio-1.5.11-go1.11.5-linux_amd64"
				source = "https://github.com/fabiolb/fabio/releases/download/v1.5.13/fabio-1.5.13-go1.13.4-linux_amd64"
				options {
					checksum = "sha256:716aaa264e2ffb7a98a574220e0e20d7d40e2f1b2717584d6f260e01f89220fc"
				}
			}

			vault {


@@ 70,3 73,5 @@ job "fabio" {
		}
	}
}

// vim: set tabstop=4 shiftwidth=4:

A policies/acme-renewer.hcl => policies/acme-renewer.hcl +3 -0
@@ 0,0 1,3 @@
path "secret/data/fabio/certs/*" {
  capabilities = ["create", "update"]
}

M scripts/README.md => scripts/README.md +2 -1
@@ 1,1 1,2 @@
This folder contains various utility scripts, intended to be installed to /usr/local/bin.
This folder contains various utility scripts for cluster members, intended to
be installed to /usr/local/bin.

M stackscripts/cluster-member.sh => stackscripts/cluster-member.sh +3 -0
@@ 181,6 181,9 @@ if [[ "${CONSUL_VERSION:-}" != "" ]]; then install_hashicorp consul "${CONSUL_VE
if [[ "${NOMAD_VERSION:-}" != "" ]]; then install_hashicorp nomad "${NOMAD_VERSION}"; fi
if [[ "${VAULT_VERSION:-}" != "" ]]; then install_hashicorp vault "${VAULT_VERSION}"; fi

# Ensure certs dirs exist so that CAs can be provisioned.
mkdir -p /etc/ssl/{consul,nomad,vault}

echo "Stackscript complete"
touch /root/StackScript.complete


M terraform/consul-server/main.tf => terraform/consul-server/main.tf +2 -2
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].b64_url}"
  label            = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 8,7 8,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].b64_url}"
    hostname       = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    consul_version = random_id.servers[count.index].keepers.consul_version
  }


M terraform/main.tf => terraform/main.tf +2 -388
@@ 50,6 50,7 @@ module "nomad-server" {
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
	vault_token      = var.vault_token
}

module "nomad-client" {


@@ 57,7 58,7 @@ module "nomad-client" {

  clients           = 2
  consul_version    = "1.7.2"
  nomad_version     = "0.11.1"
  nomad_version     = "0.9.7"
  consul_server_ips = module.consul-server.ips

  datacenter       = local.region


@@ 85,390 86,3 @@ module "vault-server" {
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

// resource "linode_instance" "ca_central_nomad_servers" {
// 	count = 1
//     region = "ca-central"
//     label = "nomad-server-ca-central-${count.index+1}"
//     tags = ["consul", "nomad"]
// 
//     image = local.image
//     type = local.instance_type
//     authorized_keys = local.authorized_keys
// 	// stackscript_id = local.stackscript_id
//     swap_size = 256
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/environment.hcl"
// 		content = <<-EOT
// 		datacenter = "ca-central"
// 		node_name = "nomad-server-ca-central-${count.index+1}"
// 		EOT
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/client.hcl"
// 		content = "server = false\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/join.hcl"
// 		content = <<-EOT
// 		retry_join = [
// 			%{for ip in linode_instance.ca_central_consul_servers[*].ipv6}
// 				${split("/", ip)[0]},
// 			%{endfor}
// 		]
// 		EOT
// 	}
// 
//  	provisioner "remote-exec" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
//  		inline = [
// 			"cat /etc/ssl/cfssl.json | CA_HOST=\"${var.ca_host}\" CA_KEY=\"${var.ca_key}\" envsubst | sponge /etc/ssl/cfssl.json",
// 			"cd /etc/ssl/consul && issue-cert.sh consul consul && chown consul:consul *",
// 			"systemctl enable consul; service consul start",
// 		]
//  	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/nomad.d/environment.hcl"
// 		content = "region = \"ca\"\ndatacenter = \"ca-central\"\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/nomad.d/server.hcl"
// 		content = "server {\n\tenabled = true\n\tbootstrap_expect = 1\n}\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/nomad.d/vault.hcl"
// 		content = "vault {\n\ttoken = \"${var.vault_token}\"\n}\n"
// 	}
// 
//  	provisioner "remote-exec" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
//  		}
//  		inline = [
// 			"hostnamectl set-hostname nomad-server-ca-central-${count.index+1}",
// 			"cat /etc/ssl/cfssl.json | CA_HOST=\"${var.ca_host}\" CA_KEY=\"${var.ca_key}\" envsubst | sponge /etc/ssl/cfssl.json",
// 			"cd /etc/ssl/nomad && issue-cert.sh nomad nomad server.ca.nomad && issue-cert.sh nomad cli && issue-cert.sh vault vault && issue-cert.sh consul consul && chown nomad:nomad *",
// 			"systemctl enable nomad; service nomad start",
// 		]
//  	}
// }
// 
// output "ca_central_nomad_server_ips" {
// 	description = "Nomad server IP addresses"
// 	value = linode_instance.ca_central_nomad_servers[*].ipv6
// }
// 
// resource "linode_instance" "ca_central_nomad_clients" {
// 	count = 1
//     region = "ca-central"
//     label = "nomad-client-ca-central-${count.index+1}"
//     tags = ["consul", "nomad"]
// 
//     image = local.image
//     type = local.instance_type
//     authorized_keys = local.authorized_keys
// 	// stackscript_id = local.stackscript_id
//     swap_size = 256
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/environment.hcl"
// 		content = "datacenter = \"ca-central\"\nnode_name = \"nomad-client-ca-central-${count.index+1}\"\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/client.hcl"
// 		content = "server = false\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/join.hcl"
//  		content = "retry_join = [${join(", ", [for address in linode_instance.ca_central_consul_servers[*].ipv6: "\"${split("/", address)[0]}\""])}]\n"
// 	}
// 
//  	provisioner "remote-exec" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
//  		inline = [
// 			"cat /etc/ssl/cfssl.json | CA_HOST=\"${var.ca_host}\" CA_KEY=\"${var.ca_key}\" envsubst | sponge /etc/ssl/cfssl.json",
// 			"cd /etc/ssl/consul && issue-cert.sh consul consul && chown consul:consul *",
// 			"mkdir /var/run/consul && chown consul:consul /var/run/consul",
// 			"systemctl enable consul; service consul start",
// 		]
//  	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/nomad.d/environment.hcl"
// 		content = "region = \"ca\"\ndatacenter = \"ca-central\"\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/nomad.d/client.hcl"
// 		content = "client {\n\tenabled = true\n}\n"
// 	}
// 
//  	provisioner "remote-exec" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_nomad_clients[count.index].ipv6)[0]
//  		}
//  		inline = [
// 			"hostnamectl set-hostname nomad-client-ca-central-${count.index+1}",
// 			"cat /etc/ssl/cfssl.json | CA_HOST=\"${var.ca_host}\" CA_KEY=\"${var.ca_key}\" envsubst | sponge /etc/ssl/cfssl.json",
// 			"cd /etc/ssl/nomad && issue-cert.sh nomad nomad client.ca.nomad && issue-cert.sh nomad cli && issue-cert.sh vault vault && issue-cert.sh consul consul && chown nomad:nomad *",
// 			"systemctl enable nomad; service nomad start",
// 		]
//  	}
// }
// 
// output "ca_central_nomad_client_ips" {
// 	description = "Nomad client IP addresses"
// 	value = linode_instance.ca_central_nomad_clients[*].ipv6
// }
// 
// resource "linode_instance" "ca_central_vault_servers" {
// 	count = 1
//     region = "ca-central"
//     label = "vault-server-ca-central-${count.index+1}"
//     tags = ["consul", "vault"]
// 
//     image = local.image
//     type = local.instance_type
//     authorized_keys = local.authorized_keys
// 	// stackscript_id = local.stackscript_id
//     swap_size = 256
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_vault_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/environment.hcl"
// 		content = "datacenter = \"ca-central\"\nnode_name = \"vault-server-ca-central-${count.index+1}\"\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_vault_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/client.hcl"
// 		content = "server = false\n"
// 	}
// 
// 	provisioner "file" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_vault_servers[count.index].ipv6)[0]
//  		}
// 		destination = "/etc/consul.d/join.hcl"
//  		content = "retry_join = [${join(", ", [for address in linode_instance.ca_central_consul_servers[*].ipv6: "\"${split("/", address)[0]}\""])}]\n"
// 	}
// 
//  	provisioner "remote-exec" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_vault_servers[count.index].ipv6)[0]
//  		}
//  		inline = [
// 			"cat /etc/ssl/cfssl.json | CA_HOST=\"${var.ca_host}\" CA_KEY=\"${var.ca_key}\" envsubst | sponge /etc/ssl/cfssl.json",
// 			"cd /etc/ssl/consul && issue-cert.sh consul consul && chown consul:consul *",
// 			"systemctl enable consul; service consul start",
// 		]
//  	}
// 
//  	provisioner "remote-exec" {
//  		connection {
//  			host = split("/", linode_instance.ca_central_vault_servers[count.index].ipv6)[0]
//  		}
//  		inline = [
// 			"hostnamectl set-hostname vault-server-ca-central-${count.index+1}",
// 			"cat /etc/ssl/cfssl.json | CA_HOST=\"${var.ca_host}\" CA_KEY=\"${var.ca_key}\" envsubst | sponge /etc/ssl/cfssl.json",
// 			"cd /etc/ssl/vault && issue-cert.sh vault vault && issue-cert.sh vault cli && issue-cert.sh consul consul && chown vault:vault *",
// 			"systemctl enable vault; service vault start",
// 		]
//  	}
// }

// ===== IGNORE EVERYTHING BELOW THIS LINE =====

// resource "linode_instance" "ca_central_nomad_servers" {
// 	count = 1
//     region = "ca-central"
//     label = "nomad-server-ca-central-${count.index+1}"
//     tags = ["consul", "nomad"]
// 
//     image = local.image
//     type = local.instance_type
//     authorized_keys = local.authorized_keys
// 	stackscript_id = local.stackscript_id
//     swap_size = 256
// 
// 	stackscript_data = {
// 		role = "nomad-server"
// 		hostname = "nomad-server-ca-central-${count.index+1}"
// 		minio_host = var.minio_host
// 		minio_access_key = var.minio_access_key
// 		minio_secret_key = var.minio_secret_key
// 		ca_server = var.ca_server
// 		ca_key = var.ca_key
// 		api_token = "something bogus" // TODO: to be (hopefully) eventually used for cluster autodiscovery
// 		consul_version = var.consul_version
// 		nomad_version = var.nomad_version
// 		vault_version = var.vault_version
// 	}
// 
// 	// Wait until the stackscript has created the consul config directory.
// 	provisioner "remote-exec" {
// 		connection {
// 			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
// 		}
// 		inline = ["until [[ -d /etc/consul.d ]]; do sleep 1; done"]
// 	}
// 
// 	provisioner "file" {
// 		connection {
// 			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
// 		}
// 		# Linode returns the IPv6 address with a "/64" suffix, which we need to trim off in order for Consul to connect.
// 		content = "retry_join = [${join(", ", [for address in linode_instance.ca_central_consul_servers[*].ipv6: "\"${split("/", address)[0]}\""])}]\n"
// 		destination = "/etc/consul.d/join.hcl"
// 	}
// }
// 
// output "ca_central_nomad_server_ips" {
// 	description = "Nomad server IP addresses"
// 	value = linode_instance.ca_central_nomad_servers[*].ipv6
// }
// 
// resource "linode_instance" "ca_central_nomad_clients" {
// 	count = 1
//     region = "ca-central"
//     label = "nomad-client-ca-central-${count.index+1}"
//     tags = ["consul", "nomad"]
// 
//     image = local.image
//     type = local.instance_type
//     authorized_keys = local.authorized_keys
// 	stackscript_id = local.stackscript_id
//     swap_size = 256
// 
// 	stackscript_data = {
// 		role = "nomad-client"
// 		hostname = "nomad-client-ca-central-${count.index+1}"
// 		minio_host = var.minio_host
// 		minio_access_key = var.minio_access_key
// 		minio_secret_key = var.minio_secret_key
// 		ca_server = var.ca_server
// 		ca_key = var.ca_key
// 		api_token = "something bogus" // TODO: to be (hopefully) eventually used for cluster autodiscovery
// 		consul_version = var.consul_version
// 		nomad_version = var.nomad_version
// 		vault_version = var.vault_version
// 	}
// 
// 	provisioner "remote-exec" {
// 		connection {
// 			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
// 		}
// 		inline = ["until [[ -d /etc/consul.d ]]; do sleep 1; done"]
// 	}
// 
// 	provisioner "file" {
// 		connection {
// 			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
// 		}
// 		# Linode returns the IPv6 address with a "/64" suffix, which we need to trim off in order for Consul to connect.
// 		content = "retry_join = [${join(", ", [for address in linode_instance.ca_central_consul_servers[*].ipv6: "\"${split("/", address)[0]}\""])}]\n"
// 		destination = "/etc/consul.d/join.hcl"
// 	}
// }
// 
// output "ca_central_nomad_client_ips" {
// 	description = "Nomad client IP addresses"
// 	value = linode_instance.ca_central_nomad_clients[*].ipv6
// }
// 
// resource "linode_instance" "ca_central_vault_servers" {
// 	count = 1
//     region = "ca-central"
//     label = "vault-server-ca-central-${count.index+1}"
//     tags = ["consul", "vault"]
// 
//     image = local.image
//     type = local.instance_type
//     authorized_keys = local.authorized_keys
// 	stackscript_id = local.stackscript_id
//     swap_size = 256
// 
// 	stackscript_data = {
// 		role = "vault-server"
// 		hostname = "vault-server-ca-central-${count.index+1}"
// 		minio_host = var.minio_host
// 		minio_access_key = var.minio_access_key
// 		minio_secret_key = var.minio_secret_key
// 		ca_server = var.ca_server
// 		ca_key = var.ca_key
// 		api_token = "something bogus" // TODO: to be (hopefully) eventually used for cluster autodiscovery
// 		consul_version = var.consul_version
// 		vault_version = var.vault_version
// 		vault_version = var.vault_version
// 	}
// 
// 	provisioner "remote-exec" {
// 		connection {
// 			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
// 		}
// 		inline = ["until [[ -d /etc/consul.d ]]; do sleep 1; done"]
// 	}
// 
// 	provisioner "file" {
// 		connection {
// 			host = split("/", linode_instance.ca_central_nomad_servers[count.index].ipv6)[0]
// 		}
// 		# Linode returns the IPv6 address with a "/64" suffix, which we need to trim off in order for Consul to connect.
// 		content = "retry_join = [${join(", ", [for address in linode_instance.ca_central_consul_servers[*].ipv6: "\"${split("/", address)[0]}\""])}]\n"
// 		destination = "/etc/consul.d/join.hcl"
// 	}
// }
// 
// output "ca_central_vault_server_ips" {
// 	description = "Vault server IP addresses"
// 	value = linode_instance.ca_central_vault_servers[*].ipv6
// }

M terraform/nomad-client/main.tf => terraform/nomad-client/main.tf +14 -2
@@ 1,6 1,6 @@
resource "linode_instance" "clients" {
  count            = var.clients
  label            = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].b64_url}"
  label            = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${replace(random_id.clients[count.index].b64_url, "-", "_")}"
  region           = random_id.clients[count.index].keepers.datacenter
  image            = random_id.clients[count.index].keepers.image
  type             = random_id.clients[count.index].keepers.instance_type


@@ 8,7 8,7 @@ resource "linode_instance" "clients" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].b64_url}"
    hostname       = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${replace(random_id.clients[count.index].b64_url, "-", "_")}"
    consul_version = random_id.clients[count.index].keepers.consul_version
    nomad_version  = random_id.clients[count.index].keepers.nomad_version
  }


@@ 86,6 86,12 @@ resource "linode_instance" "clients" {
					"/var/lib/ca-certificates/pem" = "/var/lib/ca-certificates/pem"
				}
			}

			plugin "raw_exec" {
				config {
					enabled = true
				}
			}
    EOT
  }



@@ 144,6 150,12 @@ resource "linode_instance" "clients" {
    EOT
  }

  // install additional base packages needed for running tasks
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["zypper --non-interactive install git docker"]
  }

  // reload firewall
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }

M terraform/nomad-server/main.tf => terraform/nomad-server/main.tf +18 -3
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].b64_url}"
  label            = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 8,7 8,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].b64_url}"
    hostname       = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    nomad_version  = random_id.servers[count.index].keepers.nomad_version
  }


@@ 138,6 138,21 @@ resource "linode_instance" "servers" {
    inline = ["service firewalld reload"]
  }

	// set Vault token
	provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
		inline = [
			<<-EOC
				SYSTEMD_EDITOR=tee systemctl edit nomad <<EOF
				[Service]
				Environment=VAULT_TOKEN=${var.vault_token}
				EOF
			EOC
			,
			"chmod 0400 /etc/systemd/system/nomad.service.d/override.conf"
		]
	}

  // fix permissions
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }


@@ 154,7 169,7 @@ resource "linode_instance" "servers" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames server.global.nomad,${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames nomad.service.consul,server.global.nomad,${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name cli",
      "/usr/local/bin/issue-cert.sh --user nomad --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca vault --name vault",

M terraform/nomad-server/variables.tf => terraform/nomad-server/variables.tf +1 -0
@@ 8,6 8,7 @@ variable authorized_users { type = list(string) }

variable ca_host { type = string }
variable ca_key { type = string }
variable vault_token { type = string }

variable consul_version { type = string }
variable nomad_version { type = string }

M terraform/vault-server/main.tf => terraform/vault-server/main.tf +3 -3
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].b64_url}"
  label            = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 8,7 8,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].b64_url}"
    hostname       = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    vault_version  = random_id.servers[count.index].keepers.vault_version
  }


@@ 147,7 147,7 @@ resource "linode_instance" "servers" {
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user vault --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user vault --ca vault --name vault --hostnames ${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user vault --ca vault --name vault --hostnames vault.service.consul,active.vault.service.consul,${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user vault --ca vault --name cli",
    ]
  }

A tools/README.md => tools/README.md +1 -0
@@ 0,0 1,1 @@
This folder contains tools for working within this infrastructure.

A tools/nomad-compile => tools/nomad-compile +17 -0
@@ 0,0 1,17 @@
#!/usr/bin/env ruby

require 'erb'

if ARGV.length != 1
  puts "usage: nomad-compile <job.nomad.erb>"
  exit 1
end

def secret(path, name)
  raise 'upwards traversal disallowed' if path.include?('..')
  `vault kv get -field #{name} secret/#{path}`
end

ERB::new(File.read(ARGV.first)).run

# vim: set expandtab tabstop=2 shiftwidth=2: