~damien/infrastructure

41e11f7d9d43ebc929aa90dbe0827d12a680ae64 — Damien Radtke 23 days ago b481ba4 master
Use blue-green terraform strategy for cluster upgrades
25 files changed, 243 insertions(+), 151 deletions(-)

M README.md
M config/vault/base.hcl
M jobs/damienradtkecom.nomad.erb
M jobs/fabio.nomad.erb
M terraform/README.md
R terraform/consul-server/{main.tf => -server/main.tf}
R terraform/consul-server/{outputs.tf => -server/outputs.tf}
R terraform/consul-server/{variables.tf => -server/variables.tf}
A terraform/cluster/main.tf
R terraform/nomad-client/{main.tf => -client/main.tf}
R terraform/nomad-client/{outputs.tf => -client/outputs.tf}
R terraform/nomad-client/{variables.tf => -client/variables.tf}
R terraform/nomad-server/{main.tf => -server/main.tf}
R terraform/nomad-server/{outputs.tf => -server/outputs.tf}
R terraform/nomad-server/{variables.tf => -server/variables.tf}
A terraform/cluster/outputs.tf
R terraform/vault-server/{main.tf => -server/main.tf}
R terraform/vault-server/{outputs.tf => -server/outputs.tf}
R terraform/vault-server/{variables.tf => -server/variables.tf}
M terraform/domains.tf
M terraform/main.tf
M terraform/outputs.tf
A tools/post-apply
A tools/unseal-vaults
M tools/update-nomad-client-firewall
M README.md => README.md +6 -0
@@ 57,3 57,9 @@ setting up the Packer build to remove large unneeded packages:
```bash
rpm -qa --queryformat '%10{size} - %-25{name} \t %{version}\n' | sort -n
```

# Vault Cluster Setup

1. Initialize and unseal the vault
2. Mount a K/V v2 engine at `secret/`
3. Write the policies defined in `policies`

M config/vault/base.hcl => config/vault/base.hcl +1 -0
@@ 1,4 1,5 @@
ui = true
cluster_name = "primary"

storage "consul" {
	address = "unix:///var/run/consul/consul_https.sock"

M jobs/damienradtkecom.nomad.erb => jobs/damienradtkecom.nomad.erb +4 -3
@@ 17,6 17,10 @@ job "damienradtkecom" {
			max_parallel = 1
		}

		network {
			port "http" {}
		}

		task "server" {
			driver = "exec"
			config {


@@ 50,9 54,6 @@ job "damienradtkecom" {
			resources {
				cpu = 20  # MHz, the minimum value
				memory = 50  # MB
				network {
					port "http" {}
				}
			}

			artifact {

M jobs/fabio.nomad.erb => jobs/fabio.nomad.erb +17 -18
@@ 15,6 15,22 @@ job "fabio" {
			value     = "load-balancer"
		}

		network {
			port "ui" {
				static = 9998
			}
			port "balancer" {
				// TODO: if the below interpolation works, this doesn't need to be static
				static = 9999
			}
			port "http" {
				static = 80
			}
			port "https" {
				static = 443
			}
		}

		task "fabio" {
			driver = "exec"
			user   = "fabio"


@@ 52,15 68,6 @@ job "fabio" {

			resources {
				memory = 50  # MB
				network {
					port "ui" {
						static = 9998
					}
					port "balancer" {
						// TODO: if the below interpolation works, this doesn't need to be static
						static = 9999
					}
				}
			}
		}



@@ 69,7 76,7 @@ job "fabio" {
			config {
				command = "porter",
				args = [
					"-to", "localhost:${NOMAD_PORT_fabio_balancer}",
					"-to", "localhost:${NOMAD_PORT_balancer}",
					"-http-redirect", "photos.radtke.family=https://radtke-family.synology.me:4430/photo/",
				]
			}


@@ 84,14 91,6 @@ job "fabio" {

			resources {
				memory = 50  # MB
				network {
					port "http" {
						static = 80
					}
					port "https" {
						static = 443
					}
				}
			}
		}
	}

M terraform/README.md => terraform/README.md +6 -0
@@ 1,3 1,9 @@
## Applying

```bash
$ terraform apply -var-file secrets.tfvars && post-apply
```

## Debugging

Log level and log path can be set using the `TF_LOG` and `TF_LOG_PATH` environment variables. See https://www.terraform.io/docs/internals/debugging.html

R terraform/consul-server/main.tf => terraform/cluster/consul-server/main.tf +4 -4
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  label            = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 9,7 9,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    hostname       = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    consul_version = random_id.servers[count.index].keepers.consul_version
  }



@@ 149,11 149,11 @@ resource "random_id" "servers" {
    instance_type  = var.instance_type
    consul_version = var.consul_version
  }
  byte_length = 4
  byte_length = 3
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../config/cfssl.json")
  template = file("${path.module}/../../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key

R terraform/consul-server/outputs.tf => terraform/cluster/consul-server/outputs.tf +0 -0
R terraform/consul-server/variables.tf => terraform/cluster/consul-server/variables.tf +0 -0
A terraform/cluster/main.tf => terraform/cluster/main.tf +102 -0
@@ 0,0 1,102 @@
variable linode_token { type = string }
variable ca_host { type = string }
variable ca_key { type = string }
variable vault_token { type = string }

variable region { type = string }
variable image { type = string }
variable instance_type { type = string }
variable authorized_users { type = list(string) }

variable consul_version { type = string }
variable nomad_version { type = string }
variable vault_version { type = string }

locals {
  stackscript_id   = 535217
}

module "consul-server" {
  source = "./consul-server"

  servers        = 1
  consul_version = var.consul_version

  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "nomad-server" {
  source = "./nomad-server"

  servers           = 1
  consul_version    = var.consul_version
  nomad_version     = var.nomad_version
  consul_server_ips = module.consul-server.instances[*].ipv6

  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
  vault_token      = var.vault_token
}

module "nomad-client" {
  source = "./nomad-client"

  clients           = 1
  consul_version    = var.consul_version
  nomad_version     = var.nomad_version
  consul_server_ips = module.consul-server.instances[*].ipv6

  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "nomad-client-load-balancer" {
  source = "./nomad-client"

  clients           = 1
  consul_version    = var.consul_version
  nomad_version     = var.nomad_version
  node_class        = "load-balancer"
  consul_server_ips = module.consul-server.instances[*].ipv6

  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "vault-server" {
  source = "./vault-server"

  servers           = 1
  consul_version    = var.consul_version
  vault_version     = var.vault_version
  consul_server_ips = module.consul-server.instances[*].ipv6

  datacenter       = var.region
  image            = var.image
  instance_type    = var.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = var.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

R terraform/nomad-client/main.tf => terraform/cluster/nomad-client/main.tf +4 -4
@@ 16,7 16,7 @@ locals {

resource "linode_instance" "clients" {
  count  = var.clients
  label  = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${replace(random_id.clients[count.index].b64_url, "-", "_")}"
  label  = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
  region = random_id.clients[count.index].keepers.datacenter
  image  = random_id.clients[count.index].keepers.image
  type   = random_id.clients[count.index].keepers.instance_type


@@ 26,7 26,7 @@ resource "linode_instance" "clients" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${replace(random_id.clients[count.index].b64_url, "-", "_")}"
    hostname       = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
    consul_version = random_id.clients[count.index].keepers.consul_version
    nomad_version  = random_id.clients[count.index].keepers.nomad_version
  }


@@ 285,11 285,11 @@ resource "random_id" "clients" {
    consul_version = var.consul_version
    nomad_version  = var.nomad_version
  }
  byte_length = 4
  byte_length = 3
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../config/cfssl.json")
  template = file("${path.module}/../../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key

R terraform/nomad-client/outputs.tf => terraform/cluster/nomad-client/outputs.tf +0 -0
R terraform/nomad-client/variables.tf => terraform/cluster/nomad-client/variables.tf +0 -0
R terraform/nomad-server/main.tf => terraform/cluster/nomad-server/main.tf +4 -4
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  label            = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 9,7 9,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    hostname       = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    nomad_version  = random_id.servers[count.index].keepers.nomad_version
  }


@@ 225,11 225,11 @@ resource "random_id" "servers" {
    consul_version = var.consul_version
    nomad_version  = var.nomad_version
  }
  byte_length = 4
  byte_length = 3
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../config/cfssl.json")
  template = file("${path.module}/../../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key

R terraform/nomad-server/outputs.tf => terraform/cluster/nomad-server/outputs.tf +0 -0
R terraform/nomad-server/variables.tf => terraform/cluster/nomad-server/variables.tf +0 -0
A terraform/cluster/outputs.tf => terraform/cluster/outputs.tf +24 -0
@@ 0,0 1,24 @@
output "consul-servers" {
  description = "Consul server instances"
  value       = module.consul-server.instances
}

output "nomad-servers" {
  description = "Nomad server instances"
  value       = module.nomad-server.instances
}

output "nomad-clients" {
  description = "Nomad client instances"
  value       = module.nomad-client.instances
}

output "nomad-client-load-balancers" {
  description = "Nomad client instances for load balancing"
  value       = module.nomad-client-load-balancer.instances
}

output "vault-servers" {
  description = "Vault server instances"
  value       = module.vault-server.instances
}

R terraform/vault-server/main.tf => terraform/cluster/vault-server/main.tf +4 -4
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  label            = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type


@@ 9,7 9,7 @@ resource "linode_instance" "servers" {

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    hostname       = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    vault_version  = random_id.servers[count.index].keepers.vault_version
  }


@@ 202,11 202,11 @@ resource "random_id" "servers" {
    consul_version = var.consul_version
    vault_version  = var.vault_version
  }
  byte_length = 4
  byte_length = 3
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../config/cfssl.json")
  template = file("${path.module}/../../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key

R terraform/vault-server/outputs.tf => terraform/cluster/vault-server/outputs.tf +0 -0
R terraform/vault-server/variables.tf => terraform/cluster/vault-server/variables.tf +0 -0
M terraform/domains.tf => terraform/domains.tf +8 -6
@@ 1,40 1,42 @@
module "damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  instances = module.nomad-client-load-balancer.instances
  instances = module.cluster-blue.nomad-client-load-balancers
}

module "www-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "www"
  instances = module.nomad-client-load-balancer.instances
  instances = module.cluster-blue.nomad-client-load-balancers
}

module "photos-radtke-family" {
  source    = "./domain-address"
  domain    = "radtke.family"
  name      = "photos"
  instances = module.nomad-client-load-balancer.instances
  instances = module.cluster-blue.nomad-client-load-balancers
}

module "consul-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "consul"
  instances = [module.consul-server.instances[0]]
  instances = [module.cluster-blue.consul-servers[0]]
}

module "nomad-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "nomad"
  instances = [module.nomad-server.instances[0]]
  instances = [module.cluster-blue.nomad-servers[0]]
}

/*
module "vault-damienradtke-com" {
  source    = "./domain-address"
  domain    = "damienradtke.com"
  name      = "vault"
  instances = [module.vault-server.instances[0]]
  instances = flatten([[module.vault-server-blue.instances[0]], [module.vault-server-green.instances[0]]])
}
*/

M terraform/main.tf => terraform/main.tf +14 -100
@@ 11,108 11,22 @@ variable ca_host { type = string }
variable ca_key { type = string }
variable vault_token { type = string }

locals {
  region = "ca-central"
  # image = "private/8694776"
  image = {
    default = "linode/opensuse15.1"
  }
  instance_type    = "g6-nanode-1"
  stackscript_id   = 535217
  authorized_users = [data.linode_profile.me.username]
}

module "consul-server" {
  source = "./consul-server"

  servers        = 1
  consul_version = "1.7.2"

  datacenter       = local.region
  image            = lookup(local.image, terraform.workspace, local.image["default"])
  instance_type    = local.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "nomad-server" {
  source = "./nomad-server"

  servers           = 1
  consul_version    = "1.7.2"
  nomad_version     = "0.11.3"
  consul_server_ips = module.consul-server.instances[*].ipv6

  datacenter       = local.region
  image            = lookup(local.image, terraform.workspace, local.image["default"])
  instance_type    = local.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
  vault_token      = var.vault_token
}

module "nomad-client" {
  source = "./nomad-client"
module "cluster-blue" {
	source = "./cluster"

  clients           = 1
  consul_version    = "1.7.2"
  nomad_version     = "0.11.3"
  consul_server_ips = module.consul-server.instances[*].ipv6
	linode_token = var.linode_token
	ca_host = var.ca_host
	ca_key = var.ca_key
	vault_token = var.vault_token

  datacenter       = local.region
  image            = lookup(local.image, terraform.workspace, local.image["default"])
  instance_type    = local.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "nomad-client-load-balancer" {
  source = "./nomad-client"

  clients           = 1
  consul_version    = "1.7.2"
  nomad_version     = "0.11.3"
  node_class        = "load-balancer"
  consul_server_ips = module.consul-server.instances[*].ipv6

  datacenter       = local.region
  image            = lookup(local.image, terraform.workspace, local.image["default"])
  instance_type    = local.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

module "vault-server" {
  source = "./vault-server"

  servers           = 1
  consul_version    = "1.7.2"
  vault_version     = "1.4.0"
  consul_server_ips = module.consul-server.instances[*].ipv6
	region = "ca-central"
	image = "linode/opensuse15.2"
  instance_type    = "g6-nanode-1"
  authorized_users = [data.linode_profile.me.username]

  datacenter       = local.region
  image            = lookup(local.image, terraform.workspace, local.image["default"])
  instance_type    = local.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
  consul_version = "1.9.0"
  nomad_version  = "1.0.1"
  vault_version  = "1.6.1"
}

resource "null_resource" "nomad_client_firewall" {
  triggers = {
    ips = join(",", module.nomad-client.instances[*].ipv6)
  }

  provisioner "local-exec" {
    command = "update-nomad-client-firewall"
  }
}
// TODO: cluster-green

M terraform/outputs.tf => terraform/outputs.tf +17 -8
@@ 1,13 1,15 @@
// TODO: standardize output names?

output "consul_server_ips" {
  value = [for ip in module.consul-server.instances[*].ipv6 : split("/", ip)[0]]
  value = compact(flatten([
		[for ip in module.cluster-blue.consul-servers[*].ipv6 : split("/", ip)[0]],
	]))
}

output "nomad_server_ips" {
  value = [for ip in module.nomad-server.instances[*].ipv6 : split("/", ip)[0]]
}

output "vault_server_ips" {
  value = [for ip in module.vault-server.instances[*].ipv6 : split("/", ip)[0]]
  value = compact(flatten([
		[for ip in module.cluster-blue.nomad-servers[*].ipv6 : split("/", ip)[0]],
	]))
}

output "nomad_client_ips" {


@@ 18,7 20,14 @@ output "nomad_client_ips" {
	)
	*/
  value = flatten(concat(
    module.nomad-client.instances[*].ip_address,
    module.nomad-client-load-balancer.instances[*].ip_address,
    module.cluster-blue.nomad-clients[*].ip_address,
    module.cluster-blue.nomad-client-load-balancers[*].ip_address,
  ))
}

output "vault_server_ips" {
  value = compact(flatten([
		[for ip in module.cluster-blue.vault-servers[*].ipv6 : split("/", ip)[0]],
	]))
}


A tools/post-apply => tools/post-apply +4 -0
@@ 0,0 1,4 @@
#!/usr/bin/env bash

unseal-vaults
update-nomad-client-firewall

A tools/unseal-vaults => tools/unseal-vaults +22 -0
@@ 0,0 1,22 @@
#!/usr/bin/env bash

echo "$(tput bold)Checking Vault seals...$(tput sgr0)"

function check_seal() {
	vault status >/dev/null
	exit_code=$?
	case "${exit_code}" in
		0) return 0;;
		1) echo "error getting status"; exit 1;;
		2) return 1;;
	esac
}

for server in $(terraform output -json vault_server_ips | jq -r '.[]'); do
	export VAULT_ADDR="https://[${server}]:8200" 
	while ! check_seal "${server}"; do
		echo "unsealing: ${server}"
		vault operator unseal
		echo ''
	done
done

M tools/update-nomad-client-firewall => tools/update-nomad-client-firewall +2 -0
@@ 1,5 1,7 @@
#!/usr/bin/env bash

echo "$(tput bold)Updating Nomad client firewalls...$(tput sgr0)"

# TODO: it might be good to enhance this script to also update the confured "join" values for node configurations.

if [[ ! -f terraform.tfstate ]]; then