~damien/infrastructure

4b88a55d5653d1badead32479762c5276f536482 — Damien Radtke 2 years ago 9b835da
More code
M ca/consul-agent-ca.srl => ca/consul-agent-ca.srl +1 -1
@@ 1,1 1,1 @@
2E741F16F5701C92061B3669C7546E4A1AA2C87C
2E741F16F5701C92061B3669C7546E4A1AA2C89A

M ca/nomad-agent-ca.srl => ca/nomad-agent-ca.srl +1 -1
@@ 1,1 1,1 @@
3A95C8A7EE198C7858E58366940C5F258426065A
3A95C8A7EE198C7858E58366940C5F258426067E

M ca/vault-server-ca.srl => ca/vault-server-ca.srl +1 -1
@@ 1,1 1,1 @@
73BA2463646820941EB0EC0FDCDC6DE86EE14227
73BA2463646820941EB0EC0FDCDC6DE86EE14245

M terraform/README.md => terraform/README.md +59 -1
@@ 10,7 10,65 @@ TODO: run post-apply?

### Bootstrapping

If creating a fresh cluster, you will want to add `-var bootstrap=true` so that the Consul, Nomad, and Vault servers are provisioned with the `bootstrap_expect` config.
To create a fresh cluster, run:

```sh
$ terraform init
$ terraform apply -var bootstrap=true
```

Then SSH to the Vault server and initialize + unseal it:

```sh
$ ssh <vault_server>
$ vault operator init
$ vault operator unseal  # enough times to unseal
```

The root token will be needed to run cluster upgrades.

## Upgrading the Cluster

TODO: these steps still need to be tested

Copy the `cluster` module block into a new cluster instance, `cluster-new`. Make the required upgrade changes, including any additional outputs as needed, and then run:

```sh
$ terraform init
$ terraform apply -var vault_token=<root_token>
```

Once the new cluster is up and running, run these commands on the new relevant nodes:

```sh
# On each new Consul server
$ consul join <existing_leader>

# On each new Nomad server
# ...this step might not be necessary? It was once
$ sudo service nomad restart  # to pick up the existing cluster

# TODO: probably something for Vault, too, to unseal it
```

Once all servers are part of the cluster, old and new, run these commands on the old nodes:

```sh
# On each old Consul server
$ consul leave

# On each old Nomad client
$ nomad node drain -enable -self

# TODO: something for Vault?
```

Once it's no longer being used, remove the `cluster` module from `main.tf` and apply it. Finally, rename `cluster-new` back to `cluster`, and update the state to reflect it:

```sh
$ terraform state mv module.cluster-new module.cluster
$ terraform apply
```

## Debugging


M terraform/cluster/consul-server/outputs.tf => terraform/cluster/consul-server/outputs.tf +4 -2
@@ 1,6 1,8 @@
output "ips" {
output "instances" {
  description = "Consul server instances"
  value       = [for ip in linode_instance.servers[*].ipv6 : split("/", ip)[0]]
  value       = [for instance in linode_instance.servers[*] : {
    ip = split("/", instance.ipv6)[0]
  }]
}

// vim: set expandtab shiftwidth=2 tabstop=2:

M terraform/cluster/main.tf => terraform/cluster/main.tf +4 -4
@@ 24,7 24,7 @@ module "nomad-server" {
  source = "./nomad-server"

  servers           = 1
  consul_server_ips = module.consul-server.ips
  consul_server_ips = module.consul-server.instances[*].ip

  datacenter       = var.datacenter
  image            = var.image


@@ 38,7 38,7 @@ module "nomad-client" {
  source = "./nomad-client"

  clients           = 1
  consul_server_ips = module.consul-server.ips
  consul_server_ips = module.consul-server.instances[*].ip

  datacenter       = var.datacenter
  image            = var.image


@@ 51,7 51,7 @@ module "nomad-client-load-balancer" {

  clients           = 1
  node_class        = "load-balancer"
  consul_server_ips = module.consul-server.ips
  consul_server_ips = module.consul-server.instances[*].ip

  datacenter       = var.datacenter
  image            = var.image


@@ 63,7 63,7 @@ module "vault-server" {
  source = "./vault-server"

  servers           = 1
  consul_server_ips = module.consul-server.ips
  consul_server_ips = module.consul-server.instances[*].ip

  datacenter       = var.datacenter
  image            = var.image

M terraform/cluster/nomad-client/main.tf => terraform/cluster/nomad-client/main.tf +3 -1
@@ 79,9 79,11 @@ resource "linode_instance" "clients" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d/client.hcl"
    content     = <<-EOT
      datacenter       = "${var.datacenter}"
      name = "${self.label}"
      datacenter = "${var.datacenter}"
      client {
        enabled = true
        node_class = "${var.node_class}"
      }
    EOT
  }

M terraform/cluster/nomad-client/outputs.tf => terraform/cluster/nomad-client/outputs.tf +4 -2
@@ 1,6 1,8 @@
output "instances" {
  description = "Nomad client IP addresses"
  value       = linode_instance.clients
  description = "Nomad client instances"
  value       = [for instance in linode_instance.clients[*] : {
    ip = split("/", instance.ipv6)[0]
  }]
}

// vim: set expandtab shiftwidth=2 tabstop=2:

M terraform/cluster/nomad-server/main.tf => terraform/cluster/nomad-server/main.tf +3 -2
@@ 62,7 62,8 @@ resource "linode_instance" "servers" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d/server.hcl"
    content     = <<-EOT
      datacenter       = "${var.datacenter}"
      name = "${self.label}"
      datacenter = "${var.datacenter}"
      server {
        enabled          = true
        %{if var.bootstrap}bootstrap_expect = ${var.servers}%{endif}


@@ 75,7 76,7 @@ resource "linode_instance" "servers" {
    destination = "/etc/nomad.d/vault.hcl"
    content     = <<-EOT
      vault {
        token = "TODO"
        token = "${var.vault_token}"
      }
    EOT
  }

M terraform/cluster/nomad-server/outputs.tf => terraform/cluster/nomad-server/outputs.tf +3 -1
@@ 1,6 1,8 @@
output "instances" {
  description = "Nomad server instances"
  value       = linode_instance.servers
  value       = [for instance in linode_instance.servers[*] : {
    ip = split("/", instance.ipv6)[0]
  }]
}

// vim: set expandtab shiftwidth=2 tabstop=2:

M terraform/cluster/outputs.tf => terraform/cluster/outputs.tf +6 -12
@@ 1,26 1,20 @@
output "consul-servers" {
  description = "Consul server IP addresses"
  value       = module.consul-server.ips
  description = "Consul servers"
  value       = module.consul-server.instances
}

/*
output "nomad-servers" {
  description = "Nomad server instances"
  description = "Nomad servers"
  value       = module.nomad-server.instances
}

// TODO: group by class
output "nomad-clients" {
  description = "Nomad client instances"
  description = "Nomad clients"
  value       = module.nomad-client.instances
}

output "nomad-client-load-balancers" {
  description = "Nomad client instances for load balancing"
  value       = module.nomad-client-load-balancer.instances
}

output "vault-servers" {
  description = "Vault server instances"
  description = "Vault servers"
  value       = module.vault-server.instances
}
*/

M terraform/cluster/vault-server/outputs.tf => terraform/cluster/vault-server/outputs.tf +3 -1
@@ 1,6 1,8 @@
output "instances" {
  description = "Vault server instances"
  value       = linode_instance.servers
  value       = [for instance in linode_instance.servers[*] : {
    ip = split("/", instance.ipv6)[0]
  }]
}

// vim: set expandtab shiftwidth=2 tabstop=2:

A terraform/exec-on => terraform/exec-on +10 -0
@@ 0,0 1,10 @@
#!/usr/bin/env bash

output="$1"
shift

for ip in $(terraform output -json "${output}" | jq -r '.[].ip'); do
	command="ssh '${ip}' $@"
	echo "==> ${command}"
	eval "${command}"
done

M terraform/main.tf => terraform/main.tf +17 -1
@@ 12,6 12,10 @@ variable "bootstrap" {
  type = string
  default = false
}
variable "vault_token" {
  type = string
  default = ""
}

provider "linode" {
  token = var.linode_token


@@ 26,10 30,22 @@ module "cluster" {
  // image            = "private/14625896"
  authorized_users = [data.linode_profile.me.username]
  instance_type    = "g6-nanode-1"
  vault_token      = "root_token"
  vault_token      = var.vault_token
  bootstrap        = var.bootstrap
}

output "consul-servers" {
  value = module.cluster.consul-servers
}

output "nomad-servers" {
  value = module.cluster.nomad-servers
}

output "nomad-clients" {
  value = module.cluster.nomad-clients
}

output "vault-servers" {
  value = module.cluster.vault-servers
}