~damien/infrastructure

3bb31531238e6db22fe1af9d5c720168c438032e — Damien Radtke 11 months ago a78c6d0
Allow remote cluster access from the support server
M README.md => README.md +15 -2
@@ 1,7 1,19 @@
## Issuing Certificates

Nodes are provisioned with and run the `issue-cert.sh` script for certificates,
which uses a CFSSL remote. When on the support server, instead you can do:

```bash
$ cd ~
$ RESPONSE=$(echo '{}' | sudo `which cfssl` gencert -config /etc/ssl/cfssl.json -ca /etc/ssl/nomad/ca.pem -ca-key /etc/ssl/nomad/ca-key.pem -)
$ echo "${RESPONSE}" | jq -r .cert > nomad-cli.pem; chmod 0444 nomad-cli.pem
$ echo "${RESPONSE}" | jq -r .key > nomad-cli-key.pem; chmod 0400 nomad-cli-key.pem
```

## Querying the raw Consul API

Since Consul is configured to listen for API requests over a Unix socket,
you can use a command like this to query its HTTP API:
you can use a command like this on a Consul node to query its HTTP API:

```bash
$ curl --unix-socket /var/run/consul/consul_https.sock http:///api/v1/agent/checks


@@ 9,7 21,8 @@ $ curl --unix-socket /var/run/consul/consul_https.sock http:///api/v1/agent/chec

## Finding largest packages

Use this command to find the largest installed RPM packages, normally used when setting up the Packer build to remove large unneeded packages:
Use this command to find the largest installed RPM packages, normally used when
setting up the Packer build to remove large unneeded packages:

```bash
rpm -qa --queryformat '%10{size} - %-25{name} \t %{version}\n' | sort -n

M config/consul/base.hcl => config/consul/base.hcl +2 -1
@@ 9,7 9,8 @@ cert_file = "/etc/ssl/consul/consul.pem"
key_file = "/etc/ssl/consul/consul-key.pem"

addresses {
	https = "unix:///var/run/consul/consul_https.sock"
	# Bind HTTPS to both a local Unix socket and the advertise address.
	https = "unix:///var/run/consul/consul_https.sock {{ GetPublicInterfaces | include `type` `IPv6` | offset -1 | attr `address` }}"
}

unix_sockets {

M firewall/services/consul.xml => firewall/services/consul.xml +1 -0
@@ 6,6 6,7 @@
  <port port="8300" protocol="tcp"/>
  <port port="8301" protocol="tcp"/>
  <port port="8301" protocol="udp"/>
  <port port="8501" protocol="tcp"/>
  <port port="8600" protocol="tcp"/>
  <port port="8600" protocol="udp"/>
</service>

M jobs/README.md => jobs/README.md +19 -1
@@ 1,6 1,24 @@
This folder contains Nomad job definitions.

## Running
## Running (new way)

First, make sure your `.bashrc` is set up correctly with something like

```bash
export NOMAD_ADDR="https://[2600:3c04::f03c:92ff:fe3e:6fbe]:4646"
export NOMAD_CACERT="/etc/ssl/nomad/ca.pem"
export NOMAD_CLIENT_CERT="${HOME}/nomad-cli.pem"
export NOMAD_CLIENT_KEY="${HOME}/nomad-cli-key.pem"
```

Then you can submit jobs with the `nomad` command, or use `compile.sh` if it
requires substiution of secret values:

```bash
$ ./compile.sh job.nomad | nomad job run -
```

## Running (old way)

Jobs can be run from this repo using the `run.sh` script. Make sure that the `NOMAD_ADDR`
environment variable is set and any environment variables you need saved in `.env`, then just run:

M jobs/cat-facts.nomad => jobs/cat-facts.nomad +3 -3
@@ 1,11 1,11 @@
job "cat-facts" {
	region = "us"
	region = "global"

	datacenters = ["us-central"]
	datacenters = ["ca-central"]
	type = "batch"

	periodic {
	 	cron = "7 10 * * *"
	 	cron = "0 10 * * *"
	 	time_zone = "America/Chicago"
	}


A jobs/compile.sh => jobs/compile.sh +11 -0
@@ 0,0 1,11 @@
#!/usr/bin/env bash

DIR="$(dirname "${BASH_SOURCE[0]}")"

if [[ $# -ne 1 ]]; then
	echo "usage: $0 <job>"
	exit 1
fi

ENV="${DIR}/.env"
cat "$1" | env $(cat "${ENV}" | xargs) envsubst "$(cat "${ENV}" | awk 'BEGIN {FS="="; ORS=""} {print "${" $1 "} "}')"

M terraform/consul-server/main.tf => terraform/consul-server/main.tf +3 -1
@@ 107,7 107,9 @@ resource "linode_instance" "servers" {
  // issue certs
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul"]
    inline = [
			"/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul --hostnames ${split("/", self.ipv6)[0]}",
		]
  }

  // start services

M terraform/nomad-client/main.tf => terraform/nomad-client/main.tf +14 -0
@@ 71,6 71,20 @@ resource "linode_instance" "clients" {

      client {
				enabled = true

				# This is Nomad's default chroot + SSL certs.
				chroot_env {
					"/bin"            = "/bin"
					"/etc"            = "/etc"
					"/lib"            = "/lib"
					"/lib32"          = "/lib32"
					"/lib64"          = "/lib64"
					"/run/resolvconf" = "/run/resolvconf"
					"/sbin"           = "/sbin"
					"/usr"            = "/usr"
					# This is where SSL certs actually live on openSUSE. /etc/ssl/certs is symlinked to here
					"/var/lib/ca-certificates/pem" = "/var/lib/ca-certificates/pem"
				}
			}
    EOT
  }

M terraform/nomad-server/main.tf => terraform/nomad-server/main.tf +1 -2
@@ 154,9 154,8 @@ resource "linode_instance" "servers" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames server.global.nomad",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames server.global.nomad,${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name cli",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames server.global.nomad",
      "/usr/local/bin/issue-cert.sh --user nomad --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca vault --name vault",
    ]

M terraform/vault-server/main.tf => terraform/vault-server/main.tf +1 -1
@@ 147,7 147,7 @@ resource "linode_instance" "servers" {
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user vault --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user vault --ca vault --name vault",
      "/usr/local/bin/issue-cert.sh --user vault --ca vault --name vault --hostnames ${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user vault --ca vault --name cli",
    ]
  }