~damien/infrastructure

1d0df1f636eda20573b4392cd1aa9b966d2e2992 — Damien Radtke 2 years ago d856567
Get Consul DNS resolution working
M ca/consul-agent-ca.srl => ca/consul-agent-ca.srl +1 -1
@@ 1,1 1,1 @@
2E741F16F5701C92061B3669C7546E4A1AA2C8A4
2E741F16F5701C92061B3669C7546E4A1AA2C8BB

M ca/nomad-agent-ca.srl => ca/nomad-agent-ca.srl +1 -1
@@ 1,1 1,1 @@
3A95C8A7EE198C7858E58366940C5F258426068C
3A95C8A7EE198C7858E58366940C5F25842606AE

M ca/vault-server-ca.srl => ca/vault-server-ca.srl +1 -1
@@ 1,1 1,1 @@
73BA2463646820941EB0EC0FDCDC6DE86EE1424F
73BA2463646820941EB0EC0FDCDC6DE86EE14266

M jobs/fabio.nomad.erb => jobs/fabio.nomad.erb +13 -13
@@ 1,7 1,7 @@
<%
  @fabio_version  = "1.5.13"
  @golang_version = "1.13.4"
  @fabio_checksum = "sha256:716aaa264e2ffb7a98a574220e0e20d7d40e2f1b2717584d6f260e01f89220fc"
  @fabio_version  = "1.5.15"
  @golang_version = "1.15.5"
  @fabio_checksum = "sha256:14c7a02ca95fb00a4f3010eab4e3c0e354a3f4953d2a793cb800332012f42066"
%>

job "fabio" {


@@ 12,7 12,7 @@ job "fabio" {
	group "fabio" {
		constraint {
			attribute = "${node.class}"
			value     = "load-balancer"
			value     = "ingress"
		}

		network {


@@ 35,16 35,16 @@ job "fabio" {
			driver = "exec"
			user   = "fabio"
			config {
				command = "fabio-<%= @fabio_version %>-go<%= @golang_version %>-linux_amd64",
				command = "fabio-<%= @fabio_version %>-go<%= @golang_version %>-linux_amd64"
				// TODO: this currently fails because consul-key.pem is not readable
				// Need to figure out a way to get fabio a client cert
				args = [
					"-registry.consul.addr", "https://localhost:8501",
					"-registry.consul.tls.cafile", "/etc/ssl/consul/ca.pem",
					"-registry.consul.tls.certfile", "/etc/ssl/fabio/consul.pem",
					"-registry.consul.tls.keyfile", "/etc/ssl/fabio/consul-key.pem",
					"-proxy.cs", "cs=mycerts;type=vault;cert=secret/fabio/certs",
					"-proxy.addr", ":${NOMAD_PORT_balancer};cs=mycerts",
                    "-registry.consul.tls.cafile", "/etc/ssl/consul/ca.pem",
                    "-registry.consul.tls.certfile", "/etc/ssl/fabio/consul.pem",
                    "-registry.consul.tls.keyfile", "/etc/ssl/fabio/consul-key.pem",
                    "-proxy.cs", "cs=mycerts;type=vault;cert=secret/fabio/certs",
                    "-proxy.addr", ":${NOMAD_PORT_balancer};cs=mycerts",
				]
			}



@@ 74,10 74,10 @@ job "fabio" {
		task "porter" {
			driver = "raw_exec"
			config {
				command = "porter",
				command = "porter"
				args = [
					"-to", "localhost:${NOMAD_PORT_balancer}",
					"-http-redirect", "photos.radtke.family=https://radtke-family.synology.me:4430/photo/",
                    "-to", "localhost:${NOMAD_PORT_balancer}",
                    "-http-redirect", "photos.radtke.family=https://radtke-family.synology.me:4430/photo/",
				]
			}


A packer/config/dns/consul.conf => packer/config/dns/consul.conf +5 -0
@@ 0,0 1,5 @@
zone "consul" IN {
        type forward;
        forward only;
        forwarders { 127.0.0.1 port 8600; };
};

M packer/image.pkr.hcl => packer/image.pkr.hcl +23 -10
@@ 33,7 33,16 @@ variable "vault_version" {
}

locals {
  packages = ["wget", "jq", "firewalld", "moreutils", "git"]
  zypper_packages = [
    "wget",
    "jq",
    "firewalld",
    "moreutils",
    "git",
  ]
  zypper_patterns = [
    "dhcp_dns_server",
  ]
  timestamp = regex_replace(timestamp(), "[- TZ:]", "")
}



@@ 59,7 68,9 @@ build {

  provisioner "shell" {
    inline = [
      "zypper --non-interactive install ${join(" ", local.packages)}",
      "zypper --non-interactive refresh",
      "zypper --non-interactive install ${join(" ", local.zypper_packages)}",
      "zypper --non-interactive install --type pattern ${join(" ", local.zypper_patterns)}",
      "zypper --non-interactive clean",
      "echo Updating CA certificates",
      "update-ca-certificates --verbose",


@@ 130,21 141,23 @@ build {
    source      = "../ca/vault-server-ca.crt"
  }

  /*
  provisioner "file" {
    destination = "/etc/ssl/nomad/ca.pem"
    source      = "certs/nomad-ca.pem"
    destination = "/etc/profile.local"
    source      = "config/profile.local"
  }

  provisioner "file" {
    destination = "/etc/ssl/vault/ca.pem"
    source      = "certs/vault-ca.pem"
    destination = "/etc/named.d/consul.conf"
    source      = "config/dns/consul.conf"
  }
  */

  provisioner "file" {
    destination = "/etc/profile.local"
    source      = "config/profile.local"
    destination = "/root/bin/on-startup.sh"
    source      = "scripts/on-startup.sh"
  }

  provisioner "shell" {
    script = "scripts/configure-dns.sh"
  }

  provisioner "shell" {

A packer/scripts/configure-dns.sh => packer/scripts/configure-dns.sh +17 -0
@@ 0,0 1,17 @@
#!/usr/bin/env bash
#
# This script configures DNS to forward .consul requests to Consul.
#

named_conf_include_files="$(cat /etc/sysconfig/named | grep 'NAMED_CONF_INCLUDE_FILES=' | cut -d= -f2 | tr -d '"')"
if [[ "${named_conf_include_files}" != *"consul.conf"* ]]; then
	sudo yast2 sysconfig set "NAMED_CONF_INCLUDE_FILES=${named_conf_include_files} consul.conf"
fi

yast2 sysconfig set NETCONFIG_DNS_FORWARDER=bind

# Disable DNSSEC in bind
sed -i 's/#dnssec-validation [a-z]\+;$/dnssec-validation no;/' /etc/named.conf

systemctl enable named
systemctl enable on-startup

M packer/scripts/install-hashicorp.sh => packer/scripts/install-hashicorp.sh +1 -0
@@ 1,3 1,4 @@
#!/usr/bin/env bash

osarch="linux_amd64"


A packer/scripts/on-startup.sh => packer/scripts/on-startup.sh +12 -0
@@ 0,0 1,12 @@
#!/usr/bin/env bash

orig_etc_resolv="/etc/.resolv.conf.linode-orig"
if [[ -f "${orig_etc_resolv}" ]]; then
	# Get the configured list of nameservers in a format that can be used by BIND.
	nameservers="$(cat "${orig_etc_resolv}" | grep '^nameserver ' | awk '{print $2}' | sed 's/$/;/' | xargs echo)"

	echo "Defining list of DNS forwarders as: ${nameservers}"
	sed -i "s/#\?forwarders {.*};/forwarders { ${nameservers} };/" /etc/named.conf
fi

netconfig update -f

A packer/services/on-startup.service => packer/services/on-startup.service +10 -0
@@ 0,0 1,10 @@
[Unit]
Description=Run post-boot startup actions

[Service]
Type=oneshot
Environment=TERM=dumb
ExecStart=/root/bin/on-startup.sh

[Install]
WantedBy=multi-user.target

M terraform/README.md => terraform/README.md +24 -1
@@ 17,7 17,7 @@ $ terraform init
$ terraform apply -var bootstrap=true
```

Then SSH to the Vault server and initialize + unseal it:
SSH to the Vault server and initialize + unseal it:

```sh
$ ssh <vault_server>


@@ 25,6 25,19 @@ $ vault operator init
$ vault operator unseal  # enough times to unseal
```

While you're here, mount the KV secrets engine:

```sh
$ vault secrets enable -path=secret kv-v2
```

SSH to any Nomad servers and configure them with the root Vault token:

```sh
$ sudo vim /etc/nomad.d/vault.hcl # only readable as root
$ sudo service nomad restart
```

The root token will be needed to run cluster upgrades.

## Upgrading the Cluster


@@ 70,6 83,16 @@ $ terraform state mv module.cluster-new module.cluster
$ terraform apply
```

## In-Place Upgrade

If you just want to do an in-place upgrade, which is simpler but more destructive, be sure to taint the null resource as well. Without re-applying the null resource, the new node will likely not know how to connect to Consul.

For example, to fully recreate the Nomad servers:

```sh
$ terraform state list | grep module.nomad-server | xargs -n 1 terraform taint
```

## Debugging

Log level and log path can be set using the `TF_LOG` and `TF_LOG_PATH` environment variables. See https://www.terraform.io/docs/internals/debugging.html

M terraform/consul-server/main.tf => terraform/consul-server/main.tf +2 -6
@@ 21,10 21,6 @@ resource "linode_instance" "servers" {
  group            = terraform.workspace
  tags            = [terraform.workspace]

  lifecycle {
    create_before_destroy = true
  }

  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["echo SSH is up"]


@@ 33,11 29,11 @@ resource "linode_instance" "servers" {
  // Recognize the new server
  // This isn't _ideal_, but it's better than disabling host key checking for every SSH command.
  provisioner "local-exec" {
    command = "ssh-keygen -R '${self.ip_address}' && ssh-keyscan -v '${self.ip_address}' >> ~/.ssh/known_hosts"
    command = "ssh-keygen -R '${split("/", self.ipv6)[0]}' && ssh-keyscan -v '${split("/", self.ipv6)[0]}' >> ~/.ssh/known_hosts"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca consul-agent --hostname server.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca consul-agent --hostname server.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
  }

  provisioner "file" {

M terraform/main.tf => terraform/main.tf +1 -1
@@ 54,7 54,7 @@ module "nomad-server" {
  consul_server_ips = module.consul-server.instances[*].ip

  datacenter       = local.datacenter
  image            = local.image
  image            = "private/15446344"
  instance_type    = local.instance_type
  authorized_users = local.authorized_users
  vault_token      = var.vault_token

M terraform/nomad-client/main.tf => terraform/nomad-client/main.tf +10 -9
@@ 34,10 34,6 @@ resource "linode_instance" "clients" {
  group            = terraform.workspace
  tags            = [terraform.workspace]

  lifecycle {
    create_before_destroy = true
  }

  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["echo SSH is up"]


@@ 46,11 42,11 @@ resource "linode_instance" "clients" {
  // Recognize the new server
  // This isn't _ideal_, but it's better than disabling host key checking for every SSH command.
  provisioner "local-exec" {
    command = "ssh-keygen -R '${self.ip_address}' && ssh-keyscan -v '${self.ip_address}' >> ~/.ssh/known_hosts"
    command = "ssh-keygen -R '${split("/", self.ipv6)[0]}' && ssh-keyscan -v '${split("/", self.ipv6)[0]}' >> ~/.ssh/known_hosts"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca consul-agent --hostname client.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca consul-agent --hostname client.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
  }

  provisioner "file" {


@@ 64,15 60,15 @@ resource "linode_instance" "clients" {
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca nomad-agent --hostname client.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename nomad"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca nomad-agent --hostname client.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename nomad"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca nomad-agent --owner damien:nobody --outdir /etc/ssl/nomad-agent --basename cli"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca nomad-agent --owner damien:nobody --outdir /etc/ssl/nomad-agent --basename cli"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca vault-server --hostname client.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename vault"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca vault-server --hostname client.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename vault"
  }

  provisioner "file" {


@@ 85,6 81,11 @@ resource "linode_instance" "clients" {
        enabled = true
        node_class = "${var.node_class}"
      }
      plugin "raw_exec" {
        config {
          enabled = true
        }
      }
    EOT
  }


M terraform/nomad-server/main.tf => terraform/nomad-server/main.tf +8 -9
@@ 17,10 17,6 @@ resource "linode_instance" "servers" {
  group            = terraform.workspace
  tags            = [terraform.workspace]

  lifecycle {
    create_before_destroy = true
  }

  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["echo SSH is up"]


@@ 29,15 25,15 @@ resource "linode_instance" "servers" {
  // Recognize the new server
  // This isn't _ideal_, but it's better than disabling host key checking for every SSH command.
  provisioner "local-exec" {
    command = "ssh-keygen -R '${self.ip_address}' && ssh-keyscan -v '${self.ip_address}' >> ~/.ssh/known_hosts"
    command = "ssh-keygen -R '${split("/", self.ipv6)[0]}' && ssh-keyscan -v '${split("/", self.ipv6)[0]}' >> ~/.ssh/known_hosts"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca consul-agent --hostname client.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca consul-agent --hostname client.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca vault-server --hostname server.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename vault"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca vault-server --hostname server.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename vault"
  }

  provisioner "file" {


@@ 51,11 47,11 @@ resource "linode_instance" "servers" {
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca nomad-agent --hostname server.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename nomad"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca nomad-agent --hostname server.global.nomad --owner nomad:nomad --outdir /etc/ssl/nomad-agent --basename nomad"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca nomad-agent --owner damien:nobody --outdir /etc/ssl/nomad-agent --basename cli"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca nomad-agent --owner damien:nobody --outdir /etc/ssl/nomad-agent --basename cli"
  }

  provisioner "file" {


@@ 77,6 73,9 @@ resource "linode_instance" "servers" {
    content     = <<-EOT
      vault {
        token = "${var.vault_token}"
        ca_file = "/etc/ssl/vault-server/ca.crt"
        cert_file = "/etc/ssl/nomad-agent/vault.crt"
        key_file = "/etc/ssl/nomad-agent/vault.key"
      }
    EOT
  }

M terraform/ssh => terraform/ssh +16 -0
@@ 1,1 1,17 @@
#!/usr/bin/env bash

cd "$(dirname "${BASH_SOURCE[0]}")"

if [[ $# -eq 0 ]]; then
	echo "Available outputs:"
	echo ""
	terraform output -json | jq -r 'keys | .[]'
	exit 1
fi

output="$1"; shift
addr="$(terraform output -json "${output}" | jq -r '.[0].ip')"

echo "> Connecting to ${addr}..."

exec ssh "${addr}" "$@"

M terraform/vault-server/main.tf => terraform/vault-server/main.tf +4 -8
@@ 17,10 17,6 @@ resource "linode_instance" "servers" {
  group            = terraform.workspace
  tags            = [terraform.workspace]

  lifecycle {
    create_before_destroy = true
  }

  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["echo SSH is up"]


@@ 29,11 25,11 @@ resource "linode_instance" "servers" {
  // Recognize the new server
  // This isn't _ideal_, but it's better than disabling host key checking for every SSH command.
  provisioner "local-exec" {
    command = "ssh-keygen -R '${self.ip_address}' && ssh-keyscan -v '${self.ip_address}' >> ~/.ssh/known_hosts"
    command = "ssh-keygen -R '${split("/", self.ipv6)[0]}' && ssh-keyscan -v '${split("/", self.ipv6)[0]}' >> ~/.ssh/known_hosts"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca consul-agent --hostname client.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca consul-agent --hostname client.${self.region}.consul --owner consul:consul --outdir /etc/ssl/consul-agent --basename consul"
  }

  provisioner "file" {


@@ 47,11 43,11 @@ resource "linode_instance" "servers" {
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca vault-server --hostname server.${self.region}.vault --owner vault:vault --outdir /etc/ssl/vault-server --basename vault"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca vault-server --hostname server.${self.region}.vault --owner vault:vault --outdir /etc/ssl/vault-server --basename vault"
  }

  provisioner "local-exec" {
    command = "../ca/provision-cert --addr ${self.ip_address} --ca vault-server --owner damien:nobody --outdir /etc/ssl/vault-server --basename cli"
    command = "../ca/provision-cert --addr ${split("/", self.ipv6)[0]} --ca vault-server --owner damien:nobody --outdir /etc/ssl/vault-server --basename cli"
  }

  provisioner "file" {