M README.md => README.md +6 -0
@@ 57,3 57,9 @@ setting up the Packer build to remove large unneeded packages:
```bash
rpm -qa --queryformat '%10{size} - %-25{name} \t %{version}\n' | sort -n
```
+
+# Vault Cluster Setup
+
+1. Initialize and unseal the vault
+2. Mount a K/V v2 engine at `secret/`
+3. Write the policies defined in `policies`
M config/vault/base.hcl => config/vault/base.hcl +1 -0
@@ 1,4 1,5 @@
ui = true
+cluster_name = "primary"
storage "consul" {
address = "unix:///var/run/consul/consul_https.sock"
M jobs/damienradtkecom.nomad.erb => jobs/damienradtkecom.nomad.erb +4 -3
@@ 17,6 17,10 @@ job "damienradtkecom" {
max_parallel = 1
}
+ network {
+ port "http" {}
+ }
+
task "server" {
driver = "exec"
config {
@@ 50,9 54,6 @@ job "damienradtkecom" {
resources {
cpu = 20 # MHz, the minimum value
memory = 50 # MB
- network {
- port "http" {}
- }
}
artifact {
M jobs/fabio.nomad.erb => jobs/fabio.nomad.erb +17 -18
@@ 15,6 15,22 @@ job "fabio" {
value = "load-balancer"
}
+ network {
+ port "ui" {
+ static = 9998
+ }
+ port "balancer" {
+ // TODO: if the below interpolation works, this doesn't need to be static
+ static = 9999
+ }
+ port "http" {
+ static = 80
+ }
+ port "https" {
+ static = 443
+ }
+ }
+
task "fabio" {
driver = "exec"
user = "fabio"
@@ 52,15 68,6 @@ job "fabio" {
resources {
memory = 50 # MB
- network {
- port "ui" {
- static = 9998
- }
- port "balancer" {
- // TODO: if the below interpolation works, this doesn't need to be static
- static = 9999
- }
- }
}
}
@@ 69,7 76,7 @@ job "fabio" {
config {
command = "porter",
args = [
- "-to", "localhost:${NOMAD_PORT_fabio_balancer}",
+ "-to", "localhost:${NOMAD_PORT_balancer}",
"-http-redirect", "photos.radtke.family=https://radtke-family.synology.me:4430/photo/",
]
}
@@ 84,14 91,6 @@ job "fabio" {
resources {
memory = 50 # MB
- network {
- port "http" {
- static = 80
- }
- port "https" {
- static = 443
- }
- }
}
}
}
M terraform/README.md => terraform/README.md +6 -0
@@ 1,3 1,9 @@
+## Applying
+
+```bash
+$ terraform apply -var-file secrets.tfvars && post-apply
+```
+
## Debugging
Log level and log path can be set using the `TF_LOG` and `TF_LOG_PATH` environment variables. See https://www.terraform.io/docs/internals/debugging.html
R terraform/consul-server/main.tf => terraform/cluster/consul-server/main.tf +4 -4
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
count = var.servers
- label = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
+ label = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
region = random_id.servers[count.index].keepers.datacenter
image = random_id.servers[count.index].keepers.image
type = random_id.servers[count.index].keepers.instance_type
@@ 9,7 9,7 @@ resource "linode_instance" "servers" {
stackscript_id = var.stackscript_id
stackscript_data = {
- hostname = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
+ hostname = "consul-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
consul_version = random_id.servers[count.index].keepers.consul_version
}
@@ 149,11 149,11 @@ resource "random_id" "servers" {
instance_type = var.instance_type
consul_version = var.consul_version
}
- byte_length = 4
+ byte_length = 3
}
data "template_file" "cfssl_config" {
- template = file("${path.module}/../../config/cfssl.json")
+ template = file("${path.module}/../../../config/cfssl.json")
vars = {
ca_host = var.ca_host
ca_key = var.ca_key
R terraform/consul-server/outputs.tf => terraform/cluster/consul-server/outputs.tf +0 -0
R terraform/consul-server/variables.tf => terraform/cluster/consul-server/variables.tf +0 -0
A terraform/cluster/main.tf => terraform/cluster/main.tf +102 -0
@@ 0,0 1,102 @@
+variable linode_token { type = string }
+variable ca_host { type = string }
+variable ca_key { type = string }
+variable vault_token { type = string }
+
+variable region { type = string }
+variable image { type = string }
+variable instance_type { type = string }
+variable authorized_users { type = list(string) }
+
+variable consul_version { type = string }
+variable nomad_version { type = string }
+variable vault_version { type = string }
+
+locals {
+ stackscript_id = 535217
+}
+
+module "consul-server" {
+ source = "./consul-server"
+
+ servers = 1
+ consul_version = var.consul_version
+
+ datacenter = var.region
+ image = var.image
+ instance_type = var.instance_type
+ stackscript_id = local.stackscript_id
+ authorized_users = var.authorized_users
+ ca_host = var.ca_host
+ ca_key = var.ca_key
+}
+
+module "nomad-server" {
+ source = "./nomad-server"
+
+ servers = 1
+ consul_version = var.consul_version
+ nomad_version = var.nomad_version
+ consul_server_ips = module.consul-server.instances[*].ipv6
+
+ datacenter = var.region
+ image = var.image
+ instance_type = var.instance_type
+ stackscript_id = local.stackscript_id
+ authorized_users = var.authorized_users
+ ca_host = var.ca_host
+ ca_key = var.ca_key
+ vault_token = var.vault_token
+}
+
+module "nomad-client" {
+ source = "./nomad-client"
+
+ clients = 1
+ consul_version = var.consul_version
+ nomad_version = var.nomad_version
+ consul_server_ips = module.consul-server.instances[*].ipv6
+
+ datacenter = var.region
+ image = var.image
+ instance_type = var.instance_type
+ stackscript_id = local.stackscript_id
+ authorized_users = var.authorized_users
+ ca_host = var.ca_host
+ ca_key = var.ca_key
+}
+
+module "nomad-client-load-balancer" {
+ source = "./nomad-client"
+
+ clients = 1
+ consul_version = var.consul_version
+ nomad_version = var.nomad_version
+ node_class = "load-balancer"
+ consul_server_ips = module.consul-server.instances[*].ipv6
+
+ datacenter = var.region
+ image = var.image
+ instance_type = var.instance_type
+ stackscript_id = local.stackscript_id
+ authorized_users = var.authorized_users
+ ca_host = var.ca_host
+ ca_key = var.ca_key
+}
+
+module "vault-server" {
+ source = "./vault-server"
+
+ servers = 1
+ consul_version = var.consul_version
+ vault_version = var.vault_version
+ consul_server_ips = module.consul-server.instances[*].ipv6
+
+ datacenter = var.region
+ image = var.image
+ instance_type = var.instance_type
+ stackscript_id = local.stackscript_id
+ authorized_users = var.authorized_users
+ ca_host = var.ca_host
+ ca_key = var.ca_key
+}
R terraform/nomad-client/main.tf => terraform/cluster/nomad-client/main.tf +4 -4
@@ 16,7 16,7 @@ locals {
resource "linode_instance" "clients" {
count = var.clients
- label = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${replace(random_id.clients[count.index].b64_url, "-", "_")}"
+ label = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
region = random_id.clients[count.index].keepers.datacenter
image = random_id.clients[count.index].keepers.image
type = random_id.clients[count.index].keepers.instance_type
@@ 26,7 26,7 @@ resource "linode_instance" "clients" {
stackscript_id = var.stackscript_id
stackscript_data = {
- hostname = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${replace(random_id.clients[count.index].b64_url, "-", "_")}"
+ hostname = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].hex}"
consul_version = random_id.clients[count.index].keepers.consul_version
nomad_version = random_id.clients[count.index].keepers.nomad_version
}
@@ 285,11 285,11 @@ resource "random_id" "clients" {
consul_version = var.consul_version
nomad_version = var.nomad_version
}
- byte_length = 4
+ byte_length = 3
}
data "template_file" "cfssl_config" {
- template = file("${path.module}/../../config/cfssl.json")
+ template = file("${path.module}/../../../config/cfssl.json")
vars = {
ca_host = var.ca_host
ca_key = var.ca_key
R terraform/nomad-client/outputs.tf => terraform/cluster/nomad-client/outputs.tf +0 -0
R terraform/nomad-client/variables.tf => terraform/cluster/nomad-client/variables.tf +0 -0
R terraform/nomad-server/main.tf => terraform/cluster/nomad-server/main.tf +4 -4
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
count = var.servers
- label = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
+ label = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
region = random_id.servers[count.index].keepers.datacenter
image = random_id.servers[count.index].keepers.image
type = random_id.servers[count.index].keepers.instance_type
@@ 9,7 9,7 @@ resource "linode_instance" "servers" {
stackscript_id = var.stackscript_id
stackscript_data = {
- hostname = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
+ hostname = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
consul_version = random_id.servers[count.index].keepers.consul_version
nomad_version = random_id.servers[count.index].keepers.nomad_version
}
@@ 225,11 225,11 @@ resource "random_id" "servers" {
consul_version = var.consul_version
nomad_version = var.nomad_version
}
- byte_length = 4
+ byte_length = 3
}
data "template_file" "cfssl_config" {
- template = file("${path.module}/../../config/cfssl.json")
+ template = file("${path.module}/../../../config/cfssl.json")
vars = {
ca_host = var.ca_host
ca_key = var.ca_key
R terraform/nomad-server/outputs.tf => terraform/cluster/nomad-server/outputs.tf +0 -0
R terraform/nomad-server/variables.tf => terraform/cluster/nomad-server/variables.tf +0 -0
A terraform/cluster/outputs.tf => terraform/cluster/outputs.tf +24 -0
@@ 0,0 1,24 @@
+output "consul-servers" {
+ description = "Consul server instances"
+ value = module.consul-server.instances
+}
+
+output "nomad-servers" {
+ description = "Nomad server instances"
+ value = module.nomad-server.instances
+}
+
+output "nomad-clients" {
+ description = "Nomad client instances"
+ value = module.nomad-client.instances
+}
+
+output "nomad-client-load-balancers" {
+ description = "Nomad client instances for load balancing"
+ value = module.nomad-client-load-balancer.instances
+}
+
+output "vault-servers" {
+ description = "Vault server instances"
+ value = module.vault-server.instances
+}
R terraform/vault-server/main.tf => terraform/cluster/vault-server/main.tf +4 -4
@@ 1,6 1,6 @@
resource "linode_instance" "servers" {
count = var.servers
- label = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
+ label = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
region = random_id.servers[count.index].keepers.datacenter
image = random_id.servers[count.index].keepers.image
type = random_id.servers[count.index].keepers.instance_type
@@ 9,7 9,7 @@ resource "linode_instance" "servers" {
stackscript_id = var.stackscript_id
stackscript_data = {
- hostname = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
+ hostname = "vault-server-${random_id.servers[count.index].keepers.datacenter}-${random_id.servers[count.index].hex}"
consul_version = random_id.servers[count.index].keepers.consul_version
vault_version = random_id.servers[count.index].keepers.vault_version
}
@@ 202,11 202,11 @@ resource "random_id" "servers" {
consul_version = var.consul_version
vault_version = var.vault_version
}
- byte_length = 4
+ byte_length = 3
}
data "template_file" "cfssl_config" {
- template = file("${path.module}/../../config/cfssl.json")
+ template = file("${path.module}/../../../config/cfssl.json")
vars = {
ca_host = var.ca_host
ca_key = var.ca_key
R terraform/vault-server/outputs.tf => terraform/cluster/vault-server/outputs.tf +0 -0
R terraform/vault-server/variables.tf => terraform/cluster/vault-server/variables.tf +0 -0
M terraform/domains.tf => terraform/domains.tf +8 -6
@@ 1,40 1,42 @@
module "damienradtke-com" {
source = "./domain-address"
domain = "damienradtke.com"
- instances = module.nomad-client-load-balancer.instances
+ instances = module.cluster-blue.nomad-client-load-balancers
}
module "www-damienradtke-com" {
source = "./domain-address"
domain = "damienradtke.com"
name = "www"
- instances = module.nomad-client-load-balancer.instances
+ instances = module.cluster-blue.nomad-client-load-balancers
}
module "photos-radtke-family" {
source = "./domain-address"
domain = "radtke.family"
name = "photos"
- instances = module.nomad-client-load-balancer.instances
+ instances = module.cluster-blue.nomad-client-load-balancers
}
module "consul-damienradtke-com" {
source = "./domain-address"
domain = "damienradtke.com"
name = "consul"
- instances = [module.consul-server.instances[0]]
+ instances = [module.cluster-blue.consul-servers[0]]
}
module "nomad-damienradtke-com" {
source = "./domain-address"
domain = "damienradtke.com"
name = "nomad"
- instances = [module.nomad-server.instances[0]]
+ instances = [module.cluster-blue.nomad-servers[0]]
}
+/*
module "vault-damienradtke-com" {
source = "./domain-address"
domain = "damienradtke.com"
name = "vault"
- instances = [module.vault-server.instances[0]]
+ instances = flatten([[module.vault-server-blue.instances[0]], [module.vault-server-green.instances[0]]])
}
+*/
M terraform/main.tf => terraform/main.tf +14 -100
@@ 11,108 11,22 @@ variable ca_host { type = string }
variable ca_key { type = string }
variable vault_token { type = string }
-locals {
- region = "ca-central"
- # image = "private/8694776"
- image = {
- default = "linode/opensuse15.1"
- }
- instance_type = "g6-nanode-1"
- stackscript_id = 535217
- authorized_users = [data.linode_profile.me.username]
-}
-
-module "consul-server" {
- source = "./consul-server"
-
- servers = 1
- consul_version = "1.7.2"
-
- datacenter = local.region
- image = lookup(local.image, terraform.workspace, local.image["default"])
- instance_type = local.instance_type
- stackscript_id = local.stackscript_id
- authorized_users = local.authorized_users
- ca_host = var.ca_host
- ca_key = var.ca_key
-}
-
-module "nomad-server" {
- source = "./nomad-server"
-
- servers = 1
- consul_version = "1.7.2"
- nomad_version = "0.11.3"
- consul_server_ips = module.consul-server.instances[*].ipv6
-
- datacenter = local.region
- image = lookup(local.image, terraform.workspace, local.image["default"])
- instance_type = local.instance_type
- stackscript_id = local.stackscript_id
- authorized_users = local.authorized_users
- ca_host = var.ca_host
- ca_key = var.ca_key
- vault_token = var.vault_token
-}
-
-module "nomad-client" {
- source = "./nomad-client"
+module "cluster-blue" {
+ source = "./cluster"
- clients = 1
- consul_version = "1.7.2"
- nomad_version = "0.11.3"
- consul_server_ips = module.consul-server.instances[*].ipv6
+ linode_token = var.linode_token
+ ca_host = var.ca_host
+ ca_key = var.ca_key
+ vault_token = var.vault_token
- datacenter = local.region
- image = lookup(local.image, terraform.workspace, local.image["default"])
- instance_type = local.instance_type
- stackscript_id = local.stackscript_id
- authorized_users = local.authorized_users
- ca_host = var.ca_host
- ca_key = var.ca_key
-}
-
-module "nomad-client-load-balancer" {
- source = "./nomad-client"
-
- clients = 1
- consul_version = "1.7.2"
- nomad_version = "0.11.3"
- node_class = "load-balancer"
- consul_server_ips = module.consul-server.instances[*].ipv6
-
- datacenter = local.region
- image = lookup(local.image, terraform.workspace, local.image["default"])
- instance_type = local.instance_type
- stackscript_id = local.stackscript_id
- authorized_users = local.authorized_users
- ca_host = var.ca_host
- ca_key = var.ca_key
-}
-
-module "vault-server" {
- source = "./vault-server"
-
- servers = 1
- consul_version = "1.7.2"
- vault_version = "1.4.0"
- consul_server_ips = module.consul-server.instances[*].ipv6
+ region = "ca-central"
+ image = "linode/opensuse15.2"
+ instance_type = "g6-nanode-1"
+ authorized_users = [data.linode_profile.me.username]
- datacenter = local.region
- image = lookup(local.image, terraform.workspace, local.image["default"])
- instance_type = local.instance_type
- stackscript_id = local.stackscript_id
- authorized_users = local.authorized_users
- ca_host = var.ca_host
- ca_key = var.ca_key
+ consul_version = "1.9.0"
+ nomad_version = "1.0.1"
+ vault_version = "1.6.1"
}
-resource "null_resource" "nomad_client_firewall" {
- triggers = {
- ips = join(",", module.nomad-client.instances[*].ipv6)
- }
-
- provisioner "local-exec" {
- command = "update-nomad-client-firewall"
- }
-}
+// TODO: cluster-green
M terraform/outputs.tf => terraform/outputs.tf +17 -8
@@ 1,13 1,15 @@
+// TODO: standardize output names?
+
output "consul_server_ips" {
- value = [for ip in module.consul-server.instances[*].ipv6 : split("/", ip)[0]]
+ value = compact(flatten([
+ [for ip in module.cluster-blue.consul-servers[*].ipv6 : split("/", ip)[0]],
+ ]))
}
output "nomad_server_ips" {
- value = [for ip in module.nomad-server.instances[*].ipv6 : split("/", ip)[0]]
-}
-
-output "vault_server_ips" {
- value = [for ip in module.vault-server.instances[*].ipv6 : split("/", ip)[0]]
+ value = compact(flatten([
+ [for ip in module.cluster-blue.nomad-servers[*].ipv6 : split("/", ip)[0]],
+ ]))
}
output "nomad_client_ips" {
@@ 18,7 20,14 @@ output "nomad_client_ips" {
)
*/
value = flatten(concat(
- module.nomad-client.instances[*].ip_address,
- module.nomad-client-load-balancer.instances[*].ip_address,
+ module.cluster-blue.nomad-clients[*].ip_address,
+ module.cluster-blue.nomad-client-load-balancers[*].ip_address,
))
}
+
+output "vault_server_ips" {
+ value = compact(flatten([
+ [for ip in module.cluster-blue.vault-servers[*].ipv6 : split("/", ip)[0]],
+ ]))
+}
+
A tools/post-apply => tools/post-apply +4 -0
@@ 0,0 1,4 @@
+#!/usr/bin/env bash
+
+unseal-vaults
+update-nomad-client-firewall
A tools/unseal-vaults => tools/unseal-vaults +22 -0
@@ 0,0 1,22 @@
+#!/usr/bin/env bash
+
+echo "$(tput bold)Checking Vault seals...$(tput sgr0)"
+
+function check_seal() {
+ vault status >/dev/null
+ exit_code=$?
+ case "${exit_code}" in
+ 0) return 0;;
+ 1) echo "error getting status"; exit 1;;
+ 2) return 1;;
+ esac
+}
+
+for server in $(terraform output -json vault_server_ips | jq -r '.[]'); do
+ export VAULT_ADDR="https://[${server}]:8200"
+ while ! check_seal "${server}"; do
+ echo "unsealing: ${server}"
+ vault operator unseal
+ echo ''
+ done
+done
M tools/update-nomad-client-firewall => tools/update-nomad-client-firewall +2 -0
@@ 1,5 1,7 @@
#!/usr/bin/env bash
+echo "$(tput bold)Updating Nomad client firewalls...$(tput sgr0)"
+
# TODO: it might be good to enhance this script to also update the confured "join" values for node configurations.
if [[ ! -f terraform.tfstate ]]; then