~damien/infrastructure

133009eaf2d128274acd390ccad04cc2d4b2fb4f — Damien Radtke 10 months ago 7ade7c9
Better domain and dynamic port handling for Nomad clients
10 files changed, 123 insertions(+), 76 deletions(-)

A firewall/services/nomad-dynamic-ports.xml
M firewall/zones/internal.xml
A firewall/zones/nomad-clients.xml
R jobs/{damienradtkecom.nomad => damienradtkecom.nomad.erb}
R terraform/{nomad-client/domains.tf => domains.tf}
M terraform/main.tf
M terraform/nomad-client/main.tf
M terraform/nomad-client/outputs.tf
A terraform/outputs.tf
A terraform/update-nomad-client-firewall.sh
A firewall/services/nomad-dynamic-ports.xml => firewall/services/nomad-dynamic-ports.xml +7 -0
@@ 0,0 1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<service>
  <short>Nomad dynamic ports</short>
  <description>Dynamic port range for Nomad client jobs. See https://www.nomadproject.io/docs/job-specification/network/#dynamic-ports</description>
  <port port="20000-32000" protocol="tcp"/>
  <port port="20000-32000" protocol="udp"/>
</service>

M firewall/zones/internal.xml => firewall/zones/internal.xml +2 -11
@@ 1,15 1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<zone>
  <short>Public</short>
  <description>For use in public areas. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted.</description>
  <short>Internal</short>
  <description>For internal datacenter communication.</description>
  <source address="192.168.0.0/16"/>
  <service name="ssh"/>
  <!-- TODO: are these other ones needed? -->
  <service name="mdns"/>
  <service name="samba-client"/>
  <service name="dhcpv6-client"/>

  <service name="consul"/>
  <service name="nomad"/>
  <service name="vault"/>
</zone>

A firewall/zones/nomad-clients.xml => firewall/zones/nomad-clients.xml +6 -0
@@ 0,0 1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<zone>
  <short>Nomad Clients</short>
  <description>Nomad clients should trust each other.</description>
  <service name="nomad-dynamic-ports"/>
</zone>

R jobs/damienradtkecom.nomad => jobs/damienradtkecom.nomad.erb +7 -1
@@ 7,6 7,12 @@ job "damienradtkecom" {
	group "server" {
		count = 2

		constraint {
			attribute = "${node.class}"
			operator  = "!="
			value     = "load-balancer"
		}

		update {
			max_parallel = 1
		}


@@ 53,7 59,7 @@ job "damienradtkecom" {
				source = "github.com/dradtke/blog"
				destination = "local/blog/"
				options {
					ref = "f0789dadf40e8c910e63be28af371b727d19413f"
					ref = "<%= ENV['REF'] || 'master' %>"
				}
			}


R terraform/nomad-client/domains.tf => terraform/domains.tf +16 -10
@@ 3,33 3,39 @@ data "linode_domain" "damienradtkecom" {
}

resource "linode_domain_record" "damienradtkecom_root_a" {
	count = var.node_class == "load-balancer" ? 1 : 0
	for_each = toset([
		for ip in flatten(module.nomad-client-load-balancer.instances[*].ipv4[*]):
			ip if substr(ip, 0, 8) != "192.168."  // doesn't look like terraform supports "starts with"
	])
	domain_id = data.linode_domain.damienradtkecom.id
	name = "@"
	name = ""
	record_type = "A"
	target = linode_instance.clients[0].ipv4
	target = each.value
}

resource "linode_domain_record" "damienradtkecom_root_aaaa" {
	count = var.node_class == "load-balancer" ? 1 : 0
	for_each = toset(module.nomad-client-load-balancer.instances[*].ipv6)
	domain_id = data.linode_domain.damienradtkecom.id
	name = "@"
	name = ""
	record_type = "AAAA"
	target = linode_instance.clients[0].ipv6
	target = split("/", each.value)[0]
}

resource "linode_domain_record" "damienradtkecom_www_a" {
	count = var.node_class == "load-balancer" ? 1 : 0
	for_each = toset([
		for ip in flatten(module.nomad-client-load-balancer.instances[*].ipv4[*]):
			ip if substr(ip, 0, 8) != "192.168."  // doesn't look like terraform supports "starts with"
	])
	domain_id = data.linode_domain.damienradtkecom.id
	name = "www"
	record_type = "A"
	target = linode_instance.clients[0].ipv4
	target = each.value
}

resource "linode_domain_record" "damienradtkecom_www_aaaa" {
	count = var.node_class == "load-balancer" ? 1 : 0
	for_each = toset(module.nomad-client-load-balancer.instances[*].ipv6)
	domain_id = data.linode_domain.damienradtkecom.id
	name = "www"
	record_type = "AAAA"
	target = linode_instance.clients[0].ipv6
	target = split("/", each.value)[0]
}

M terraform/main.tf => terraform/main.tf +1 -1
@@ 56,7 56,7 @@ module "nomad-server" {
module "nomad-client" {
  source = "./nomad-client"

  clients           = 2
  clients           = 1
  consul_version    = "1.7.2"
  nomad_version     = "0.9.7"
  consul_server_ips = module.consul-server.ips

M terraform/nomad-client/main.tf => terraform/nomad-client/main.tf +54 -51
@@ 1,17 1,17 @@
locals {
	extra_provisions_for_class = {
		"load-balancer" = [
			"groupadd fabio",
  extra_provisions_for_class = lookup({
    "load-balancer" = [
      "groupadd fabio",
      "useradd --gid fabio fabio",
      "mkdir -p --mode=0500 /etc/ssl/fabio",
			"chown fabio:fabio /etc/ssl/fabio",
      "chown fabio:fabio /etc/ssl/fabio",
      "/usr/local/bin/issue-cert.sh --user fabio --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user fabio --ca vault --name vault",
			"firewall-cmd --zone=public --add-service=fabio --permanent",
			"firewall-cmd --zone=public --add-service=web --permanent",
			"firewall-cmd --reload",
		]
	}
      "firewall-cmd --zone=public --add-service=fabio --permanent",
      "firewall-cmd --zone=public --add-service=web --permanent",
      "firewall-cmd --reload",
    ]
  }, var.node_class, [])
}

resource "linode_instance" "clients" {


@@ 20,6 20,7 @@ resource "linode_instance" "clients" {
  region           = random_id.clients[count.index].keepers.datacenter
  image            = random_id.clients[count.index].keepers.image
  type             = random_id.clients[count.index].keepers.instance_type
  // private_ip       = true
  authorized_users = var.authorized_users

  stackscript_id = var.stackscript_id


@@ 58,6 59,7 @@ resource "linode_instance" "clients" {
    source      = "../config/consul"
  }

  // consul client config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d/client.hcl"


@@ 79,43 81,44 @@ resource "linode_instance" "clients" {
    source      = "../config/nomad"
  }

  // nomad client config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d/client.hcl"
    content     = <<-EOT
		  datacenter = "${var.datacenter}"
      datacenter = "${var.datacenter}"

      client {
				enabled = true

				# This is Nomad's default chroot + SSL certs.
				chroot_env {
					"/bin"            = "/bin"
					"/etc"            = "/etc"
					"/lib"            = "/lib"
					"/lib32"          = "/lib32"
					"/lib64"          = "/lib64"
					"/run/resolvconf" = "/run/resolvconf"
					"/sbin"           = "/sbin"
					"/usr"            = "/usr"
					# This is where SSL certs actually live on openSUSE. /etc/ssl/certs is symlinked to here
					"/var/lib/ca-certificates/pem" = "/var/lib/ca-certificates/pem"
				}

				node_class = "${var.node_class}"

				meta {
					%{for key, value in var.meta~}
					"${key}" = "${value}"
					%{endfor~}
				}
			}

			plugin "raw_exec" {
				config {
					enabled = true
				}
			}
        enabled = true

        # This is Nomad's default chroot + SSL certs.
        chroot_env {
          "/bin"            = "/bin"
          "/etc"            = "/etc"
          "/lib"            = "/lib"
          "/lib32"          = "/lib32"
          "/lib64"          = "/lib64"
          "/run/resolvconf" = "/run/resolvconf"
          "/sbin"           = "/sbin"
          "/usr"            = "/usr"
          # This is where SSL certs actually live on openSUSE. /etc/ssl/certs is symlinked to here
          "/var/lib/ca-certificates/pem" = "/var/lib/ca-certificates/pem"
        }

        node_class = "${var.node_class}"

        meta {
          %{for key, value in var.meta~}
          "${key}" = "${value}"
          %{endfor~}
        }
      }

      plugin "raw_exec" {
        config {
          enabled = true
        }
      }
    EOT
  }



@@ 167,10 170,10 @@ resource "linode_instance" "clients" {
    destination = "/etc/profile.local"
    content     = <<-EOT
      export CONSUL_HTTP_ADDR=unix:///var/run/consul/consul_https.sock
			export NOMAD_ADDR=https://localhost:4646
			export NOMAD_CACERT=/etc/ssl/nomad/ca.pem
			export NOMAD_CLIENT_CERT=/etc/ssl/nomad/cli.pem
			export NOMAD_CLIENT_KEY=/etc/ssl/nomad/cli-key.pem
      export NOMAD_ADDR=https://localhost:4646
      export NOMAD_CACERT=/etc/ssl/nomad/ca.pem
      export NOMAD_CLIENT_CERT=/etc/ssl/nomad/cli.pem
      export NOMAD_CLIENT_KEY=/etc/ssl/nomad/cli-key.pem
    EOT
  }



@@ 231,16 234,16 @@ resource "linode_instance" "clients" {
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
			"sudo -u damien /usr/local/bin/consul -autocomplete-install",
			"sudo -u damien /usr/local/bin/nomad -autocomplete-install",
		]
      "sudo -u damien /usr/local/bin/consul -autocomplete-install",
      "sudo -u damien /usr/local/bin/nomad -autocomplete-install",
    ]
  }

	// run extra provisions based on the node class
	provisioner "remote-exec" {
  // run extra provisions based on the node class
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
		inline = lookup(local.extra_provisions_for_class, var.node_class, [])
	}
    inline = local.extra_provisions_for_class
  }

  // disable further root ssh
  provisioner "remote-exec" {

M terraform/nomad-client/outputs.tf => terraform/nomad-client/outputs.tf +2 -2
@@ 1,4 1,4 @@
output "ips" {
output "instances" {
  description = "Nomad client IP addresses"
  value       = linode_instance.clients[*].ipv6
  value       = linode_instance.clients
}

A terraform/outputs.tf => terraform/outputs.tf +12 -0
@@ 0,0 1,12 @@
output "nomad_client_ips" {
	/*
	value = concat(
		[for ip in module.nomad-client.instances[*].ipv6: split("/", ip)[0]],
		[for ip in module.nomad-client-load-balancer.instances[*].ipv6: split("/", ip)[0]],
	)
	*/
	value = flatten(concat(
		module.nomad-client.instances[*].ipv4,
		module.nomad-client-load-balancer.instances[*].ipv4,
	))
}

A terraform/update-nomad-client-firewall.sh => terraform/update-nomad-client-firewall.sh +16 -0
@@ 0,0 1,16 @@
#!/usr/bin/env bash
client_ips="$(cat terraform.tfstate | jq -r '.outputs.nomad_client_ips.value[]')"
for from_ip in ${client_ips}; do
	echo "clearing existing values for ${from_ip}"
	ssh "${from_ip}" "sudo sed -i '/<source address/d' /etc/firewalld/zones/nomad-clients.xml"
	ssh "${from_ip}" "sudo firewall-cmd --reload"
	for to_ip in ${client_ips}; do
		if [[ "${from_ip}" = "${to_ip}" ]]; then
			continue
		fi
		echo "from ${from_ip} trust ${to_ip}"
		ssh "${from_ip}" "sudo firewall-cmd --zone=nomad-clients --add-source='${to_ip}' --permanent"
	done
	echo "reloading for ${from_ip}"
	ssh "${from_ip}" "sudo firewall-cmd --reload"
done