~damien/infrastructure

6e780e446771b6c7081908847fa4883f834be2f6 — Damien Radtke 11 months ago 35e2611
Add nomad-client
M scripts/issue-cert.sh => scripts/issue-cert.sh +1 -1
@@ 9,7 9,7 @@
set -o errexit
set -o pipefail

HOSTNAMES="localhost,127.0.0.1"
HOSTNAMES="localhost"

while (( "$#" )); do
	case "$1" in

M terraform/main.tf => terraform/main.tf +17 -0
@@ 52,6 52,23 @@ module "nomad-server" {
  ca_key           = var.ca_key
}

module "nomad-client" {
  source = "./nomad-client"

  clients           = 2
  consul_version    = "1.7.2"
  nomad_version     = "0.11.1"
  consul_server_ips = module.consul-server.ips

  datacenter       = local.region
  image            = local.image
  instance_type    = local.instance_type
  stackscript_id   = local.stackscript_id
  authorized_users = local.authorized_users
  ca_host          = var.ca_host
  ca_key           = var.ca_key
}

// resource "linode_instance" "ca_central_nomad_servers" {
// 	count = 1
//     region = "ca-central"

A terraform/nomad-client/main.tf => terraform/nomad-client/main.tf +217 -0
@@ 0,0 1,217 @@
resource "linode_instance" "clients" {
  count            = var.clients
  label            = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].b64_url}"
  region           = random_id.clients[count.index].keepers.datacenter
  image            = random_id.clients[count.index].keepers.image
  type             = random_id.clients[count.index].keepers.instance_type
  authorized_users = var.authorized_users

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-client-${random_id.clients[count.index].keepers.datacenter}-${random_id.clients[count.index].b64_url}"
    consul_version = random_id.clients[count.index].keepers.consul_version
    nomad_version  = random_id.clients[count.index].keepers.nomad_version
  }

  // wait for stackscript to complete
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "while ! [[ -f /root/StackScript.complete ]]; do echo 'waiting for stackscript to complete...'; sleep 3; done",
    ]
  }

  // systemd service files
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/systemd/system/"
    source      = "../services/"
  }

  // cfssl config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/cfssl.json"
    content     = data.template_file.cfssl_config.rendered
  }

  // consul config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d"
    source      = "../config/consul"
  }

  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d/client.hcl"
    content     = <<-EOT
      datacenter       = "${var.datacenter}"
      server           = false
      retry_join       = [
        %{for ip in var.consul_server_ips~}
          "${split("/", ip)[0]}",
        %{endfor~}
      ]
    EOT
  }

  // nomad config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d"
    source      = "../config/nomad"
  }

  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d/client.hcl"
    content     = <<-EOT
		  datacenter = "${var.datacenter}"

      client {
				enabled = true
			}
    EOT
  }

  // firewall services
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/firewalld/services"
    source      = "../firewall/services/"
  }

  // firewall zones
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/firewalld/zones"
    source      = "../firewall/zones/"
  }

  // issue-cert script
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/usr/local/bin/issue-cert.sh"
    source      = "../scripts/issue-cert.sh"
  }

  // Consul certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/consul/ca.pem"
    source      = "/etc/ssl/consul/ca.pem"
  }

  // Nomad certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/nomad/ca.pem"
    source      = "/etc/ssl/nomad/ca.pem"
  }

  // Vault certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/vault/ca.pem"
    source      = "/etc/ssl/vault/ca.pem"
  }

  // set global environment variables
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/profile.local"
    content     = <<-EOT
      export CONSUL_HTTP_ADDR=unix:///var/run/consul/consul_https.sock
			export NOMAD_ADDR=https://localhost:4646
			export NOMAD_CACERT=/etc/ssl/nomad/ca.pem
			export NOMAD_CLIENT_CERT=/etc/ssl/nomad/cli.pem
			export NOMAD_CLIENT_KEY=/etc/ssl/nomad/cli-key.pem
    EOT
  }

  // reload firewall
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["service firewalld reload"]
  }

  // fix permissions
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "chown -R consul:consul /etc/consul.d",
      "chown -R nomad:nomad /etc/nomad.d",
      "chmod +x /usr/local/bin/issue-cert.sh",
      "chmod 0400 /etc/ssl/cfssl.json",
    ]
  }

  // issue certs
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames client.global.nomad",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name cli",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames client.global.nomad",
      "/usr/local/bin/issue-cert.sh --user nomad --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca vault --name vault",
    ]
  }

  // fix CLI key permissions
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "chmod g+r /etc/ssl/nomad/cli-key.pem",
    ]
  }

  // start services
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "systemctl enable consul && service consul start",
      "systemctl enable nomad && service nomad start",
    ]
  }

  // install autocompletion
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
			"sudo -u damien /usr/local/bin/consul -autocomplete-install",
			"sudo -u damien /usr/local/bin/nomad -autocomplete-install",
		]
  }

  // disable further root ssh
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "sed -i 's/PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config",
      "service sshd reload",
    ]
  }
}

resource "random_id" "clients" {
  count = var.clients
  keepers = {
    datacenter     = var.datacenter
    image          = var.image
    instance_type  = var.instance_type
    consul_version = var.consul_version
    nomad_version  = var.nomad_version
  }
  byte_length = 4
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key
  }
}

A terraform/nomad-client/outputs.tf => terraform/nomad-client/outputs.tf +4 -0
@@ 0,0 1,4 @@
output "ips" {
  description = "Nomad client IP addresses"
  value       = linode_instance.clients[*].ipv6
}

A terraform/nomad-client/variables.tf => terraform/nomad-client/variables.tf +14 -0
@@ 0,0 1,14 @@
variable clients { type = number }

variable datacenter { type = string }
variable image { type = string }
variable instance_type { type = string }
variable stackscript_id { type = number }
variable authorized_users { type = list(string) }

variable ca_host { type = string }
variable ca_key { type = string }

variable consul_version { type = string }
variable nomad_version { type = string }
variable consul_server_ips { type = list(string) }

M terraform/nomad-server/main.tf => terraform/nomad-server/main.tf +2 -0
@@ 67,6 67,8 @@ resource "linode_instance" "servers" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d/server.hcl"
    content     = <<-EOT
		  datacenter = "${var.datacenter}"

      server {
				enabled = true
				bootstrap_expect = ${var.servers}