~damien/infrastructure

ref: b481ba41c3bbdf8d4f9c0393f87b495160adc414 infrastructure/terraform/nomad-server/main.tf -rw-r--r-- 6.8 KiB
b481ba41 — Damien Radtke Suppress cluster env output if non-interactive 5 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
resource "linode_instance" "servers" {
  count            = var.servers
  label            = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
  region           = random_id.servers[count.index].keepers.datacenter
  image            = random_id.servers[count.index].keepers.image
  type             = random_id.servers[count.index].keepers.instance_type
  authorized_users = var.authorized_users
  group            = terraform.workspace

  stackscript_id = var.stackscript_id
  stackscript_data = {
    hostname       = "nomad-server-${random_id.servers[count.index].keepers.datacenter}-${replace(random_id.servers[count.index].b64_url, "-", "_")}"
    consul_version = random_id.servers[count.index].keepers.consul_version
    nomad_version  = random_id.servers[count.index].keepers.nomad_version
  }

  lifecycle {
    create_before_destroy = true
  }

  // wait for stackscript to complete
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "while ! [[ -f /root/StackScript.complete ]]; do echo 'waiting for stackscript to complete...'; sleep 3; done",
    ]
  }

  // systemd service files
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/systemd/system/"
    source      = "../services/"
  }

  // cfssl config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/cfssl.json"
    content     = data.template_file.cfssl_config.rendered
  }

  // consul config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d"
    source      = "../config/consul"
  }

  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/consul.d/client.hcl"
    content     = <<-EOT
      datacenter       = "${var.datacenter}"
      server           = false
      retry_join       = [
        %{for ip in var.consul_server_ips~}
          "${split("/", ip)[0]}",
        %{endfor~}
      ]
    EOT
  }

  // nomad config
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d"
    source      = "../config/nomad"
  }

  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/nomad.d/server.hcl"
    content     = <<-EOT
      datacenter = "${var.datacenter}"

      server {
        enabled = true
        bootstrap_expect = ${var.servers}
      }
    EOT
  }

  // firewall services
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/firewalld/services"
    source      = "../firewall/services/"
  }

  // firewall zones
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/firewalld/zones"
    source      = "../firewall/zones/"
  }

  // issue-cert script
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/usr/local/bin/issue-cert.sh"
    source      = "../scripts/issue-cert.sh"
  }

  // Consul certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/consul/ca.pem"
    source      = "/etc/ssl/consul/ca.pem"
  }

  // Nomad certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/nomad/ca.pem"
    source      = "/etc/ssl/nomad/ca.pem"
  }

  // Vault certificate authority
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/ssl/vault/ca.pem"
    source      = "/etc/ssl/vault/ca.pem"
  }

  // set global environment variables
  provisioner "file" {
    connection { host = split("/", self.ipv6)[0] }
    destination = "/etc/profile.local"
    content     = <<-EOT
      export CONSUL_HTTP_ADDR=unix:///var/run/consul/consul_https.sock
      export NOMAD_ADDR=https://localhost:4646
      export NOMAD_CACERT=/etc/ssl/nomad/ca.pem
      export NOMAD_CLIENT_CERT=/etc/ssl/nomad/cli.pem
      export NOMAD_CLIENT_KEY=/etc/ssl/nomad/cli-key.pem
    EOT
  }

  // reload firewall
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = ["service firewalld reload"]
  }

  // set Vault token
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      <<-EOC
        SYSTEMD_EDITOR=tee systemctl edit nomad <<EOF
        [Service]
        Environment=VAULT_TOKEN=${var.vault_token}
        EOF
      EOC
      ,
      "chmod 0400 /etc/systemd/system/nomad.service.d/override.conf"
    ]
  }

  // fix permissions
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "chown -R consul:consul /etc/consul.d",
      "chown -R nomad:nomad /etc/nomad.d",
      "chmod +x /usr/local/bin/issue-cert.sh",
      "chmod 0400 /etc/ssl/cfssl.json",
    ]
  }

  // issue certs
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "/usr/local/bin/issue-cert.sh --user consul --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name nomad --hostnames nomad.service.consul,server.global.nomad,${split("/", self.ipv6)[0]}",
      "/usr/local/bin/issue-cert.sh --user nomad --ca nomad --name cli",
      "/usr/local/bin/issue-cert.sh --user nomad --ca consul --name consul",
      "/usr/local/bin/issue-cert.sh --user nomad --ca vault --name vault",
    ]
  }

  // fix CLI key permissions
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "chmod g+r /etc/ssl/nomad/cli-key.pem",
    ]
  }

  // start services
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "systemctl enable consul && service consul start",
      "systemctl enable nomad && service nomad start",
    ]
  }

  // install autocompletion
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "sudo -u damien /usr/local/bin/consul -autocomplete-install",
      "sudo -u damien /usr/local/bin/nomad -autocomplete-install",
    ]
  }

  // disable further root ssh
  provisioner "remote-exec" {
    connection { host = split("/", self.ipv6)[0] }
    inline = [
      "sed -i 's/PermitRootLogin .*/PermitRootLogin no/' /etc/ssh/sshd_config",
      "service sshd reload",
    ]
  }
}

resource "random_id" "servers" {
  count = var.servers
  keepers = {
    datacenter     = var.datacenter
    image          = var.image
    instance_type  = var.instance_type
    consul_version = var.consul_version
    nomad_version  = var.nomad_version
  }
  byte_length = 4
}

data "template_file" "cfssl_config" {
  template = file("${path.module}/../../config/cfssl.json")
  vars = {
    ca_host = var.ca_host
    ca_key  = var.ca_key
  }
}

// vim: set expandtab shiftwidth=2 tabstop=2: