11 files changed, 201 insertions(+), 1031 deletions(-)
M cmd/up/main.go
D cmd/up/up_test.go
D inventory.go
D lexer.go
M parser.go
D parser_test.go
D testdata/dupe_inventory
D testdata/empty
D testdata/invalid_inventory
D testdata/two_inventory_groups
M up.go
M cmd/up/main.go => cmd/up/main.go +145 -424
@@ 3,16 3,14 @@ package main
import (
"bufio"
- "bytes"
"errors"
"flag"
"fmt"
- "io"
- "io/ioutil"
"log"
"math/rand"
"os"
"os/exec"
+ "sort"
"strings"
"time"
@@ 20,46 18,38 @@ import (
)
type flags struct {
- // Upfile allows you to specify a different Upfile name. This is
+ // upfile allows you to specify a different Upfile name. This is
// helpful when running across multiple operating systems or shells.
// For example, you may have Upfile.windows.toml and Upfile.linux.toml,
// or Upfile.bash.toml and Upfile.fish.toml.
- Upfile string
+ upfile string
- // Inventory allows you specify a different inventory name.
- Inventory string
-
- // Command to run. Like `make`, an empty Command defaults to the first
+ // command to run. Like `make`, an empty Command defaults to the first
// command in the Upfile.
- Command up.CmdName
+ command string
- // Tags limits the changed services to those enumerated if the flag is
- // provided. This holds the tags that will be used.
- Tags map[string]struct{}
+ // targets are the servers (typically IP addresses) upon which up will
+ // run commands.
+ targets []string
- // Serial determines how many servers of the same type will be operated
+ // serial determines how many servers of the same type will be operated
// on at any one time. This defaults to 1. Use 0 to specify all of
- // them.
- Serial int
-
- // Vars passed into `up` at runtime to be used in start commands.
- Vars map[string]string
+ // them in parallel.
+ serial int
- // Stdin instructs `up` to read from stdin, achieved with `up -`.
- Stdin bool
+ // vars passed into `up` at runtime to be used in start commands.
+ vars map[string]string
- // Verbose will log full commands, even when they're very log. By
+ // verbose will log full commands, even when they're very long. By
// default `up` truncates commands to 80 characters when logging,
// except in the case of a failure where the full command is displayed.
- Verbose bool
+ verbose bool
- // Prompt instructs `up` to wait for input before moving onto the next
+ // prompt instructs `up` to wait for input before moving onto the next
// batch.
- Prompt bool
+ prompt bool
}
-type batch map[string][][]string
-
type result struct {
server string
err error
@@ 82,168 72,88 @@ func run() error {
return usage(fmt.Errorf("parse flags: %w", err))
}
- var upFi io.ReadCloser
- if flgs.Stdin {
- // Read until we see a NUL byte, which is an indication that we
- // are done. We can't use ReadAll or similar because we need to
- // still accept Stdin on commands for things like doas/sudo
- // prompts when Up is running.
- byt, err := bufio.NewReader(os.Stdin).ReadBytes(0)
- if err != nil && err != io.EOF {
- return fmt.Errorf("read bytes: %w", err)
- }
- // Remove the NUL byte delimiter 0
- if len(byt) >= 1 && byt[len(byt)-1] == byte(0) {
- byt = byt[:len(byt)-1]
- }
- upFi = ioutil.NopCloser(bytes.NewReader(byt))
- } else {
- upFi, err = os.Open(flgs.Upfile)
- if err != nil {
- return fmt.Errorf("open upfile: %w", err)
- }
+ upFi, err := os.Open(flgs.upfile)
+ if err != nil {
+ return fmt.Errorf("open upfile: %w", err)
}
+ defer upFi.Close()
conf, err := up.ParseUpfile(upFi)
if err != nil {
return fmt.Errorf("parse upfile: %w", err)
}
- if err = upFi.Close(); err != nil {
- return fmt.Errorf("close: %w", err)
- }
-
- // Open and parse the inventory file
- invFi, err := os.Open(flgs.Inventory)
- if err != nil {
- return fmt.Errorf("open inventory: %w", err)
- }
- defer invFi.Close()
- inventory, err := up.ParseInventory(invFi)
- if err != nil {
- return fmt.Errorf("parse inventory: %w", err)
- }
- if flgs.Command != "" && flgs.Upfile != "-" {
- conf.DefaultCommand = flgs.Command
+ if flgs.command == "" {
+ flgs.command = conf.DefaultCommand
if _, exist := conf.Commands[conf.DefaultCommand]; !exist {
- return fmt.Errorf("undefined command: %s", conf.DefaultCommand)
+ return fmt.Errorf("undefined command: %s",
+ conf.DefaultCommand)
}
}
- lims := []string{}
- for lim := range flgs.Tags {
- lims = append(lims, string(lim))
- }
- tmp := strings.Join(lims, ", ")
- if tmp == "" {
- tmp = string(conf.DefaultCommand)
- }
-
- if _, exist := inventory["all"]; exist {
- return errors.New("reserved keyword 'all' cannot be inventory name")
+ cmds := conf.Commands[flgs.command]
+ if len(cmds) == 0 {
+ return fmt.Errorf("%s has no commands", flgs.command)
}
- // Default the tags equal to the command name, which makes the
- // following work: `cup -f my_app.json | up -`
- if len(flgs.Tags) == 0 {
- flgs.Tags[string(conf.DefaultCommand)] = struct{}{}
- }
-
- // Remove any unnecessary inventory. All remaining defined inventory
- // will be used.
- if _, exist := flgs.Tags["all"]; exist {
- for ip, tags := range inventory {
- inventory[ip] = append(tags, "all")
+ for k, v := range conf.Vars {
+ if _, ok := flgs.vars[k]; !ok {
+ flgs.vars[k] = v
}
- } else {
- for ip, tags := range inventory {
- var found bool
- for _, t := range tags {
- if _, exist := flgs.Tags[t]; exist {
- found = true
- break
- }
- }
- if !found {
- delete(inventory, ip)
- }
- }
- }
-
- // Remove any tags which are not in the provided flags, as we'll be
- // ignoring those
- for ip, tags := range inventory {
- var newTags []string
- for _, t := range tags {
- if _, exist := flgs.Tags[t]; !exist {
- continue
- }
- newTags = append(newTags, t)
- }
- inventory[ip] = newTags
}
-
- // Validate all tags are defined in inventory (i.e. no silent failure
- // on typos).
- if len(inventory) == 0 {
- msg := fmt.Sprintf("tags not defined in inventory: ")
- for l := range flgs.Tags {
- msg += fmt.Sprintf("%s, ", l)
+ for i := range cmds {
+ cmds, err = substituteVariables(flgs.vars, cmds, cmds[i])
+ if err != nil {
+ return fmt.Errorf("substitute vars: %w", err)
}
- return errors.New(strings.TrimSuffix(msg, ", "))
}
- log.Printf("running %s on %s\n", conf.DefaultCommand, tmp)
-
- // Split into batches limited in size by the provided Serial flag.
- batches, err := makeBatches(conf, inventory, flgs.Serial)
+ // Split into batches limited in size by the provided serial flag.
+ batches, err := makeBatches(flgs.targets, flgs.serial)
if err != nil {
return fmt.Errorf("make batches: %w", err)
}
log.Printf("got batches: %v\n", batches)
- // For each batch, run the ExecIfs and run Execs if necessary.
- done := make(chan struct{}, len(batches))
- crash := make(chan error)
- defer close(crash)
- for _, srvBatch := range batches {
- // Schedule our next batch to run
- go func(srvBatch [][]string) {
- for i, srvGroup := range srvBatch {
- ch := make(chan result, len(srvGroup))
- srvGroup = randomizeOrder(srvGroup)
- cmd := conf.Commands[conf.DefaultCommand]
- runExecIfs(ch, flgs.Vars, conf.Commands, cmd,
- "", srvGroup, flgs.Verbose)
- for j := 0; j < len(srvGroup); j++ {
- res := <-ch
- if res.err != nil {
- crash <- res.err
- return
- }
- }
-
- // We want to prompt to continue unless it's
- // the last batch
- if flgs.Prompt && i != len(srvBatch)-1 {
- if err := confirmPrompt(srvGroup); err != nil {
- crash <- err
- return
- }
- }
+ for _, batch := range batches {
+ job := job{ips: batch, cmds: cmds}
+ ipErrs := make(chan error)
+ for _, ip := range job.ips {
+ go ipWorker(ip, job.cmds, ipErrs, flgs.verbose)
+ }
+ for range job.ips {
+ if err := <-ipErrs; err != nil {
+ return err
}
- done <- struct{}{}
- }(srvBatch)
- }
- for i := 0; i < len(batches); i++ {
- select {
- case <-done:
- // Keep going
- case err := <-crash:
- return err
}
}
return nil
}
+type job struct {
+ ips []string
+ cmds []string
+}
+
+func ipWorker(ip string, cmds []string, ipErrs chan<- error, verbose bool) {
+ for _, cmd := range cmds {
+ cmd = strings.ReplaceAll(cmd, "$server", ip)
+ logLine := fmt.Sprintf("[%s] %s", ip, cmd)
+ if !verbose && len(logLine) > 90 {
+ logLine = logLine[:87] + "..."
+ }
+ log.Printf("%s\n", logLine)
+
+ c := exec.Command("sh", "-c", cmd)
+ c.Stdout = os.Stdout
+ c.Stderr = os.Stderr
+ c.Stdin = os.Stdin
+ if err := c.Run(); err != nil {
+ ipErrs <- err
+ return
+ }
+ }
+ ipErrs <- nil
+}
+
// confirmPrompt prompts the user and asks if up should continue.
func confirmPrompt(ips []string) error {
var shouldContinue string
@@ 267,176 177,31 @@ func confirmPrompt(ips []string) error {
}
}
-func runExecIfs(
- ch chan result,
- vars map[string]string,
- cmds map[up.CmdName]*up.Cmd,
- cmd *up.Cmd,
- chk string,
- servers []string,
- verbose bool,
-) {
- send := func(ch chan<- result, err error, servers []string) {
- for _, srv := range servers {
- ch <- result{server: srv, err: err}
- }
- }
- var needToRun bool
- for _, execIf := range cmd.ExecIfs {
- // TODO should this also enforce ExecIfs? Probably...
- // TODO this should handle errors correctly through the channel
- cmds := copyCommands(cmds)
- steps := cmds[execIf].Execs
- for _, step := range steps {
- ok, err := runExec(vars, cmds, step, chk, servers,
- true, verbose)
- if err != nil {
- send(ch, err, servers)
- return
- }
- if !ok {
- needToRun = true
- }
- }
- }
- if !needToRun && len(cmd.ExecIfs) > 0 {
- for _, srv := range servers {
- ch <- result{server: srv}
- }
- return
- }
- for _, cmdLine := range cmd.Execs {
- cmdLine, err := substituteVariables(vars, cmds, cmdLine)
- if err != nil {
- send(ch, err, servers)
- return
- }
-
- // We may have substituted a variable with a multi-line command
- cmdLines := strings.SplitN(cmdLine, "\n", -1)
- for _, cmdLine := range cmdLines {
- _, err = runExec(vars, cmds, cmdLine, chk, servers,
- false, verbose)
- if err != nil {
- send(ch, err, servers)
- return
- }
- }
- }
- send(ch, nil, servers)
-}
-
-// runExec reports whether all execIfs passed and an error if any.
-func runExec(
- vars map[string]string,
- cmds map[up.CmdName]*up.Cmd,
- cmd, chk string,
- servers []string,
- execIf, verbose bool,
-) (bool, error) {
- cmds = copyCommands(cmds)
- ch := make(chan runResult, len(servers))
- for _, server := range servers {
- go runCmd(ch, vars, cmds, cmd, chk, server, execIf, verbose)
- }
- var err error
- pass := true
- for i := 0; i < len(servers); i++ {
- res := <-ch
- pass = pass && res.pass
- if res.error != nil {
- err = res.error
- }
- }
- return pass, err
-}
-
-type runResult struct {
- pass bool
- error error
-}
-
-func runCmd(
- ch chan<- runResult,
- vars map[string]string,
- cmds map[up.CmdName]*up.Cmd,
- cmd, chk, server string,
- execIf, verbose bool,
-) {
- // TODO ensure that no cycles are present with depth-first
- // search
-
- // Now substitute any variables designated by a '$'
- cmds = copyCommands(cmds)
- cmds["server"] = &up.Cmd{Execs: []string{server}}
- cmd, err := substituteVariables(vars, cmds, cmd)
- if err != nil {
- err = fmt.Errorf("substitute: %w", err)
- ch <- runResult{pass: false, error: err}
- return
- }
-
- logLine := fmt.Sprintf("[%s] %s", server, cmd)
- if !verbose && len(logLine) > 90 {
- logLine = logLine[:87] + "..."
- }
- log.Printf("%s\n", logLine)
-
- c := exec.Command("sh", "-c", cmd)
- c.Stdout = os.Stdout
- c.Stderr = os.Stderr
- c.Stdin = os.Stdin
- if err = c.Run(); err != nil {
- if execIf {
- // TODO log if verbose
- ch <- runResult{pass: false}
- return
- }
-
- fmt.Println("error running command:", cmd)
- ch <- runResult{pass: false, error: err}
- return
- }
- ch <- runResult{pass: true}
-}
-
// parseFlags and validate them.
-func parseFlags() (flags, error) {
- var (
- upfile = flag.String("f", "Upfile", "path to upfile")
- inventory = flag.String("i", "inventory.json", "path to inventory")
- command = flag.String("c", "", "command to run in upfile (use - to read from stdin)")
- tags = flag.String("t", "", "tags from inventory to run (defaults to the name of the command)")
- serial = flag.Int("n", 1, "how many of each type of server to operate on at a time")
- prompt = flag.Bool("p", false, "prompt before moving to the next batch (default false)")
- verbose = flag.Bool("v", false, "verbose logs full commands (default false)")
- )
+func parseFlags() (*flags, error) {
+ f := &flags{vars: map[string]string{}}
+ flag.StringVar(&f.upfile, "f", "Upfile", "path to upfile")
+ flag.IntVar(&f.serial, "n", 1, "how many servers to operate on at a time")
+ flag.BoolVar(&f.prompt, "p", false, "prompt before moving to the next batch")
+ flag.BoolVar(&f.verbose, "v", false, "verbose logs full commands")
+ targets := flag.String("t", "", "comma-separated targets (required)")
flag.Parse()
- if *command == "" && *upfile != "-" {
- return flags{}, errors.New("command is required")
+ args := flag.Args()
+ switch len(args) {
+ case 0: // Do nothing
+ case 1:
+ f.command = args[0]
+ default:
+ return nil, errors.New("unknown extra args")
}
-
- lim := map[string]struct{}{}
- if *tags != "" {
- lims := strings.Split(*tags, ",")
- if len(lims) > 0 {
- all := false
- for _, service := range lims {
- if service == "all" {
- lim["all"] = struct{}{}
- all = true
- }
- }
- if all && len(lims) > 1 {
- return flags{}, errors.New("cannot use 'all' tag alongside others")
- }
- for _, service := range lims {
- lim[service] = struct{}{}
- }
- }
+ if *targets == "" {
+ return nil, errors.New("-t targets is required")
}
- extraVars := map[string]string{}
+ f.targets = strings.Split(*targets, ",")
+ rand.Shuffle(len(f.targets), func(i, j int) {
+ f.targets[i], f.targets[j] = f.targets[j], f.targets[i]
+ })
for _, pair := range os.Environ() {
if len(pair) == 0 {
continue
@@ 446,52 211,39 @@ func parseFlags() (flags, error) {
if len(vals) != 2 {
continue
}
- extraVars[vals[0]] = vals[1]
- }
- flgs := flags{
- Tags: lim,
- Upfile: *upfile,
- Inventory: *inventory,
- Serial: *serial,
- Command: up.CmdName(*command),
- Vars: extraVars,
- Stdin: *upfile == "-",
- Verbose: *verbose,
- Prompt: *prompt,
+ f.vars[vals[0]] = vals[1]
}
- return flgs, nil
+ return f, nil
}
func makeBatches(
- conf *up.Config,
- inventory up.Inventory,
+ targets []string,
max int,
-) (batch, error) {
- batches := batch{}
-
- // Organize by tags, rather than IPs for efficiency in this next
- // operation
- invMap := map[string][]string{}
- for ip, tags := range inventory {
- for _, tag := range tags {
- if _, exist := invMap[tag]; !exist {
- invMap[tag] = []string{}
- }
- invMap[tag] = append(invMap[tag], ip)
- }
+) ([][]string, error) {
+ if max == 0 {
+ return [][]string{targets}, nil
}
- // Now create batches for each tag
- for tag, ips := range invMap {
- if max == 0 {
- batches[tag] = [][]string{ips}
+ batchesLen := len(targets) / max
+ if len(targets)%max != 0 {
+ // Always round up
+ batchesLen++
+ }
+ var (
+ batches = make([][]string, 0, batchesLen)
+ batch = make([]string, 0, max)
+ )
+ for _, t := range targets {
+ if i := len(batch); i < max {
+ batch = append(batch, t)
continue
}
- b := [][]string{}
- for _, ip := range ips {
- b = appendToBatch(b, ip, max)
- }
- batches[tag] = b
+ batches = append(batches, batch)
+ batch = make([]string, 0, max)
+ batch = append(batch, t)
+ }
+ if len(batch) > 0 {
+ batches = append(batches, batch)
}
if len(batches) == 0 {
return nil, errors.New("empty batches, nothing to do")
@@ 499,71 251,40 @@ func makeBatches(
return batches, nil
}
-// appendToBatch adds to the existing last batch if smaller than the max size.
-// Otherwise it creates and appends a new batch to the end.
-func appendToBatch(b [][]string, srv string, max int) [][]string {
- if len(b) == 0 {
- return [][]string{{srv}}
- }
- last := b[len(b)-1]
- if len(last) >= max {
- return append(b, []string{srv})
- }
- b[len(b)-1] = append(last, srv)
- return b
-}
-
-func randomizeOrder(ss []string) []string {
- out := make([]string, len(ss))
- perm := rand.Perm(len(ss))
- for i, p := range perm {
- out[i] = ss[p]
- }
- return out
-}
-
-// substituteVariables recursively up to 10 times. After 10 substitutions, this
-// function reports an error.
+// substituteVariables recursively up to 50 times. After 50 substitutions, this
+// function reports an error in lieu of proper cycle detection.
func substituteVariables(
vars map[string]string,
- cmds map[up.CmdName]*up.Cmd,
+ cmds []string,
cmd string,
-) (string, error) {
- replacements := []string{}
- for cmdName, cmd := range cmds {
- if len(cmd.ExecIfs) > 0 {
- continue
- }
- replacements = append(replacements, "$"+string(cmdName))
- rep := ""
- for _, c := range cmd.Execs {
- rep += c + "\n"
+) ([]string, error) {
+ // Arrange variables biggest to smallest, so $abc is substituted before
+ // $a.
+ bigToSmall := make([]string, 0, len(vars))
+ for v := range vars {
+ bigToSmall = append(bigToSmall, v)
+ }
+ sort.Slice(bigToSmall, func(i, j int) bool {
+ return len(bigToSmall[j]) < len(bigToSmall[i])
+ })
+ var lastReplaced string
+ for i := 0; i < 50; i++ {
+ var replaced bool
+ for i := range cmds {
+ orig := cmds[i]
+ for _, v := range bigToSmall {
+ cmds[i] = strings.ReplaceAll(cmds[i], "$"+v, vars[v])
+ lastReplaced = v
+ }
+ if cmds[i] != orig {
+ replaced = true
+ }
}
- rep = strings.TrimSpace(rep)
- replacements = append(replacements, rep)
- }
- for name, val := range vars {
- replacements = append(replacements, "$"+name)
- replacements = append(replacements, val)
- }
- r := strings.NewReplacer(replacements...)
- for i := 0; i < 10; i++ {
- tmp := r.Replace(cmd)
- if cmd == tmp {
- // We're done
- return cmd, nil
+ if !replaced {
+ return cmds, nil
}
- cmd = tmp
- }
- return "", errors.New("possible cycle detected")
-}
-
-func copyCommands(m1 map[up.CmdName]*up.Cmd) map[up.CmdName]*up.Cmd {
- m2 := map[up.CmdName]*up.Cmd{}
- for k, v := range m1 {
- m2[k] = v
}
- return m2
+ return nil, fmt.Errorf("possible cycle detected on %s", lastReplaced)
}
// usage prints usage instructions. It passes through any error to be sent to
@@ 575,7 296,7 @@ func usage(err error) error {
OPTIONS
[-c] command to run in upfile
- [-f] path to Upfile, default "Upfile" or use "-" to read from stdin
+ [-f] path to Upfile, default "Upfile"
[-h] short-form help with flags
[-i] path to inventory, default "inventory.json"
[-n] number of servers to execute in parallel, default 1
D cmd/up/up_test.go => cmd/up/up_test.go +0 -147
@@ 1,147 0,0 @@
-package main
-
-import (
- "fmt"
- "log"
- "testing"
-
- "egt.run/up"
-)
-
-func TestMakeBatches(t *testing.T) {
- t.Parallel()
- tcs := []struct {
- serial int
- have map[up.InvName][]string
- want batch
- }{
- {
- serial: 1,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- },
- want: batch{
- "srv1": [][]string{{"a"}, {"b"}, {"c"}},
- },
- },
- {
- serial: 3,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- "srv2": []string{"d", "e"},
- },
- want: batch{
- "srv1": [][]string{{"a", "b", "c"}},
- "srv2": [][]string{{"d", "e"}},
- },
- },
- {
- serial: 0,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- "srv2": []string{"d", "e"},
- },
- want: batch{
- "srv1": [][]string{{"a", "b", "c"}},
- "srv2": [][]string{{"d", "e"}},
- },
- },
- {
- serial: 2,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- "srv2": []string{"d", "e", "f", "g"},
- },
- want: batch{
- "srv1": [][]string{{"a", "b"}, {"c"}},
- "srv2": [][]string{{"d", "e"}, {"f", "g"}},
- },
- },
- {
- serial: 3,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- "srv2": []string{"d", "e", "f", "g"},
- },
- want: batch{
- "srv1": [][]string{{"a", "b", "c"}},
- "srv2": [][]string{{"d", "e", "f"}, {"g"}},
- },
- },
- {
- serial: 10,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- "srv2": []string{"d", "e", "f", "g"},
- },
- want: batch{
- "srv1": [][]string{{"a", "b", "c"}},
- "srv2": [][]string{{"d", "e", "f", "g"}},
- },
- },
- {
- serial: 2,
- have: map[up.InvName][]string{
- "srv1": []string{"a", "b", "c"},
- "srv2": []string{"d", "e", "f", "g"},
- "srv3": []string{"d", "e"},
- "srv4": []string{"h", "j"},
- "srv5": []string{"k", "i"},
- "srv6": []string{"l", "m", "n"},
- "srv7": []string{"o"},
- "srv8": []string{"p", "q", "r", "s", "t", "u", "v"},
- },
- want: batch{
- "srv1": [][]string{{"a", "b"}, {"c"}},
- "srv2": [][]string{{"d", "e"}, {"f", "g"}},
- "srv3": [][]string{{"d", "e"}},
- "srv4": [][]string{{"h", "j"}},
- "srv5": [][]string{{"k", "i"}},
- "srv6": [][]string{{"l", "m"}, {"n"}},
- "srv7": [][]string{{"o"}},
- "srv8": [][]string{{"p", "q"}, {"r", "s"}, {"t", "u"}, {"v"}},
- },
- },
- }
- for i, tc := range tcs {
- t.Run(fmt.Sprint(i), func(t *testing.T) {
- conf := &up.Config{Inventory: tc.have}
- batches, err := makeBatches(conf, tc.serial)
- if err != nil {
- t.Fatal(err)
- }
- for typ, ipgroups := range batches {
- wantgroups := tc.want[typ]
- if !sliceDeepEq(wantgroups, ipgroups) {
- log.Printf("%+v\n", batches)
- t.Fatalf("expected %+v, got %+v",
- tc.want[typ], ipgroups)
- }
- }
- })
- }
-}
-
-// sliceDeepEq compares nested slice equality without caring about order.
-func sliceDeepEq(a, b [][]string) bool {
- if len(a) != len(b) {
- return false
- }
- count := 0
- seen := map[string]struct{}{}
- for i, vs := range a {
- if len(vs) != len(b[i]) {
- return false
- }
- for _, v := range vs {
- count++
- seen[v] = struct{}{}
- }
- }
- for _, vs := range b {
- for _, v := range vs {
- seen[v] = struct{}{}
- }
- }
- return len(seen) == count
-}
D inventory.go => inventory.go +0 -17
@@ 1,17 0,0 @@
-package up
-
-import (
- "encoding/json"
- "fmt"
- "io"
-)
-
-type Inventory map[string][]string
-
-func ParseInventory(rdr io.Reader) (Inventory, error) {
- inv := Inventory{}
- if err := json.NewDecoder(rdr).Decode(&inv); err != nil {
- return nil, fmt.Errorf("decode: %w", err)
- }
- return inv, nil
-}
D lexer.go => lexer.go +0 -190
@@ 1,190 0,0 @@
-package up
-
-import (
- "fmt"
- "strings"
- "unicode"
- "unicode/utf8"
-)
-
-const eof = -1
-
-type tokenType int
-
-const (
- tokenError tokenType = iota // Error occurred. Value is text of err
- tokenEOF // Designate the end of the file
- tokenSpace // Run of spaces separating arguments
- tokenTab // Tab '\t'
- tokenNewline // Line break
- tokenText // Plaintext
- tokenComment // Pound '#'
-
- // Keywords follow
- tokenKeyword // Used only to delimit keywords
- tokenInventory // "inventory"
-)
-
-type token struct {
- typ tokenType
- pos int
- val string
-}
-
-type stateFn func(*lexer) stateFn
-
-// run lexes the input by executing state functions until the state is nil.
-func (l *lexer) run() {
- for l.state = lexText; l.state != nil; {
- l.state = l.state(l)
- }
- close(l.tokens) // No more tokens will be delivered
-}
-
-// lexer holds the state of the scanner.
-type lexer struct {
- input string // The string being scanned
- state stateFn // The next lexing function to enter
- start int // Start position of this token
- pos int // Current position in the input
- width int // Width of the last rune read
- lastPos int // Position of last token returned by nextToken
- tokens chan token // Channel of scanned tokens
-}
-
-func lex(input string) *lexer {
- l := &lexer{
- input: input,
- state: lexText,
- tokens: make(chan token),
- }
- go l.run()
- return l
-}
-
-// drain the output so the lexing goroutine will exit. Called by the parser,
-// not in the lexing goroutine.
-func (l *lexer) drain() {
- for range l.tokens {
- }
-}
-
-// emit passes an token back to the client.
-func (l *lexer) emit(t tokenType) {
- tkn := token{typ: t, val: l.input[l.start:l.pos]}
- l.tokens <- tkn
- l.start = l.pos
-}
-
-func (l *lexer) next() rune {
- if l.pos >= len(l.input) {
- l.width = 0
- return eof
- }
- r, w := utf8.DecodeRuneInString(l.input[l.pos:])
- l.width = w
- l.pos += l.width
- return r
-}
-
-// nextToken reports the next token from the input.
-func (l *lexer) nextToken() token {
- token := <-l.tokens
- l.lastPos = token.pos
- return token
-}
-
-// ignore skips over the pending input before this point.
-func (l *lexer) ignore() {
- l.start = l.pos
-}
-
-// backup steps back one rune. It can be called only once per call of next.
-func (l *lexer) backup() {
- l.pos -= l.width
-}
-
-// peek returns but does not consume the next rune in the input.
-func (l *lexer) peek() rune {
- r := l.next()
- l.backup()
- return r
-}
-
-// accept consumes the next rune if it's from the valid set.
-func (l *lexer) accept(valid string) bool {
- if strings.IndexRune(valid, l.next()) >= 0 {
- return true
- }
- l.backup()
- return false
-}
-
-// acceptRun consumes a run of runes from the valid set.
-func (l *lexer) acceptRun(valid string) {
- for strings.IndexRune(valid, l.next()) >= 0 {
- }
- l.backup()
-}
-
-// errorf returns an error token and terminates the scan by passing back a nil
-// pointer as the next state, terminating l.run.
-func (l *lexer) errorf(format string, args ...interface{}) stateFn {
- l.tokens <- token{typ: tokenError, val: fmt.Sprintf(format, args...)}
- return nil
-}
-
-func lexText(l *lexer) stateFn {
-Outer:
- for {
- text := l.input[l.start:l.pos]
- r := l.next()
- switch {
- case r == eof:
- break Outer
- case r == '#':
- l.emit(tokenComment)
- case text == "inventory":
- l.backup()
- l.emit(tokenInventory)
- case isEndOfLine(r):
- l.backup()
- if len(text) > 0 {
- l.emit(tokenText)
- }
- l.next()
- l.emit(tokenNewline)
- case r == ' ':
- l.backup()
- if len(text) > 0 {
- l.emit(tokenText)
- }
- return lexSpace
- case r == '\t':
- l.emit(tokenTab)
- }
- }
- // Correctly reached EOF
- if l.pos > l.start {
- l.emit(tokenText)
- }
- l.emit(tokenEOF)
- return nil
-}
-
-func lexSpace(l *lexer) stateFn {
- for l.peek() == ' ' {
- l.next()
- }
- l.emit(tokenSpace)
- return lexText
-}
-
-func isAlphaNumeric(r rune) bool {
- return r == '_' || r == '.' || unicode.IsLetter(r) ||
- unicode.IsDigit(r)
-}
-
-func isEndOfLine(r rune) bool {
- return r == '\r' || r == '\n'
-}
M parser.go => parser.go +52 -135
@@ 1,159 1,76 @@
package up
import (
+ "bufio"
"errors"
"fmt"
+ "io"
+ "strings"
)
-// parseUpfile to build a Config tree.
-func parseUpfile(text string) (*Config, error) {
+// ParseUpfile to build a Config tree.
+func ParseUpfile(r io.Reader) (*Config, error) {
t := &Config{
- Commands: map[CmdName]*Cmd{},
- text: text,
- lex: lex(text),
+ Commands: map[string][]string{},
+ Vars: map[string]string{},
}
- if err := t.parse(); err != nil {
- t.lex.drain()
- t.stopParse()
- return nil, err
- }
- t.stopParse()
- // Validate to ensure that ExecIfs are defined after fully loading
- // them, since we don't require them to be defined in a specific order
- for cmdName, cmd := range t.Commands {
- for _, execIf := range cmd.ExecIfs {
- if execIf == cmdName {
- return nil, fmt.Errorf("%s depends on itself", execIf)
- }
- if _, exist := t.Commands[execIf]; !exist {
- return nil, fmt.Errorf("%s is undefined", execIf)
- }
+ var cmd, line string
+ scn := bufio.NewScanner(r)
+ for i := 1; scn.Scan(); i++ {
+ // We might be continuing a previous line from a trailing '\'
+ curLine := scn.Text()
+ trimmedLine := strings.TrimSpace(curLine)
+ if trimmedLine == "" {
+ continue
}
- }
- if len(t.Commands) == 0 {
- return nil, errors.New("no commands")
- }
- return t, nil
-}
-
-func (t *Config) parse() error {
- return t.nextControl(t.nextNonSpace())
-}
-
-func (t *Config) stopParse() {
- t.lex = nil
-}
-
-func (t *Config) nextNonSpace() token {
- for {
- tkn := t.lex.nextToken()
- if tkn.typ != tokenSpace {
- return tkn
+ if trimmedLine[0] == '#' {
+ continue
}
- }
-}
-
-func (t *Config) nextControl(tkn token) error {
- switch tkn.typ {
- case tokenEOF:
- return nil
- default:
- return t.commandControl(CmdName(tkn.val))
- }
-}
-func (t *Config) commandControl(name CmdName) error {
- if len(t.Commands) == 0 {
- t.DefaultCommand = name
- }
- if t.Commands[name] != nil {
- return fmt.Errorf("duplicate command %s", name)
- }
- cmd := Cmd{}
-
- // Get all tokenText until newline, ignoring non-newline spaces
-Outer2:
- for {
- tkn := t.lex.nextToken()
- switch tkn.typ {
- case tokenText:
- cmd.ExecIfs = append(cmd.ExecIfs, CmdName(tkn.val))
- case tokenNewline:
- break Outer2
- case tokenSpace:
- // Do nothing
- case tokenEOF:
- return errors.New("unexpected eof in command line")
- default:
- return fmt.Errorf("unexpected command token %s (%d)", tkn.val, tkn.typ)
+ if cmd != "" && curLine[0] != '\t' {
+ cmd = ""
}
- }
- // Get all tokenText until not indented
- var indented bool
- var line string
- var tkn token
-Outer:
- for {
- tkn = t.lex.nextToken()
- switch tkn.typ {
- case tokenComment:
- skipLine(t.lex)
- indented = false
- continue
- case tokenNewline:
- indented = false
- if line != "" {
- cmd.Execs = append(cmd.Execs, line)
- line = ""
+ // We're parsing a command
+ if cmd != "" {
+ line += trimmedLine
+ if line[len(line)-1] == '\\' {
+ line = line[:len(line)-1]
+ continue
}
+ t.Commands[cmd] = append(t.Commands[cmd], line)
+ line = ""
continue
- case tokenTab:
- if indented {
- if t.lex.nextToken().typ == tokenNewline {
- t.lex.backup()
- // Ignore extra whitespace at end of lines
- continue
- }
- // But error if there are too many tabs
- // otherwise
- return errors.New("unexpected double indent")
- }
- indented = true
+ }
+
+ // We're not parsing a command. Either we're parsing a variable
+ // or a command name. Attempt to parse a variable first.
+ varParts := strings.SplitN(curLine, "=", 2)
+ if len(varParts) == 2 {
+ t.Vars[varParts[0]] = varParts[1]
continue
- case tokenText, tokenSpace:
- if !indented {
- break Outer
- }
- // Continue parsing til the end of the line
- line += tkn.val
- case tokenEOF:
- break Outer
- default:
- return fmt.Errorf("unexpected %d %q", tkn.typ, tkn.val)
}
- }
- // Ensure we found at least one
- if len(cmd.Execs) == 0 {
- return fmt.Errorf("nothing to exec for %s", name)
+ // We're not parsing a variable, so we're parsing a command
+ // name.
+ if strings.Index(curLine, ":") == -1 {
+ return nil, fmt.Errorf("line %d: missing colon in command", i)
+ }
+ parts := strings.SplitN(curLine, ":", 2)
+ cmd = parts[0]
+ if parts[1] != "" {
+ return nil, fmt.Errorf("line %d: unexpected content after colon", i)
+ }
+ if t.DefaultCommand == "" {
+ t.DefaultCommand = cmd
+ }
}
- t.Commands[name] = &cmd
- if t.DefaultCommand == "" {
- t.DefaultCommand = name
+ if err := scn.Err(); err != nil {
+ return nil, fmt.Errorf("scan: %w", err)
}
- return t.nextControl(tkn)
-}
-
-func skipLine(l *lexer) {
- for {
- tkn := l.nextToken()
- switch tkn.typ {
- case tokenNewline, tokenEOF:
- return
- default:
- continue
- }
+ if len(t.Commands) == 0 {
+ return nil, errors.New("no commands")
}
+ return t, nil
}
D parser_test.go => parser_test.go +0 -67
@@ 1,67 0,0 @@
-package up
-
-import (
- "bytes"
- "encoding/json"
- "io/ioutil"
- "path/filepath"
- "testing"
-)
-
-func TestParse(t *testing.T) {
- t.Parallel()
- tests := []struct {
- haveFile string
- want *Config
- wantErr bool
- }{
- {haveFile: "empty", wantErr: true},
- {haveFile: "dupe_inventory", wantErr: true},
- {haveFile: "invalid_inventory", wantErr: true},
- {haveFile: "two_inventory_groups", want: &Config{
- Inventory: map[InvName][]string{
- "production": []string{"1.1.1.1"},
- "staging": []string{"www.example.com", "1.1.1.2"},
- },
- Commands: map[CmdName]*Cmd{
- "deploy": &Cmd{
- ExecIfs: []CmdName{"if1"},
- Execs: []string{"echo 'hello world'"},
- },
- "if1": &Cmd{Execs: []string{"echo 'if1'"}},
- },
- DefaultCommand: "deploy",
- DefaultEnvironment: "production",
- }},
- }
- for _, tc := range tests {
- t.Run(tc.haveFile, func(t *testing.T) {
- pth := filepath.Join("testdata", tc.haveFile)
- byt, err := ioutil.ReadFile(pth)
- if err != nil {
- t.Fatal(err)
- }
- rdr := bytes.NewReader(byt)
- conf, err := Parse(rdr)
- if err != nil {
- if tc.wantErr {
- return
- }
- t.Fatal(err)
- }
- byt, err = json.Marshal(conf)
- if err != nil {
- t.Fatal(err)
- }
- got := string(byt)
- byt, err = json.Marshal(tc.want)
- if err != nil {
- t.Fatal(err)
- }
- want := string(byt)
- if got != want {
- t.Fatalf("expected: %s\ngot: %s", want, got)
- }
- })
- }
-}
D testdata/dupe_inventory => testdata/dupe_inventory +0 -5
@@ 1,5 0,0 @@
-inventory production
- 1.1.1.1
-
-inventory production
- 1.2.3.4
D testdata/empty => testdata/empty +0 -0
D testdata/invalid_inventory => testdata/invalid_inventory +0 -2
@@ 1,2 0,0 @@
-inventory
- 1.1.1.1
D testdata/two_inventory_groups => testdata/two_inventory_groups +0 -12
@@ 1,12 0,0 @@
-inventory production
- 1.1.1.1
-
-inventory staging
- www.example.com
- 1.1.1.2
-
-deploy if1
- echo 'hello world'
-
-if1
- echo 'if1'
M up.go => up.go +4 -32
@@ 1,43 1,15 @@
package up
-import (
- "fmt"
- "io"
- "io/ioutil"
-)
-
type CmdName string
// Config represents a parsed Upfile.
type Config struct {
// Commands available to run grouped by command name.
- Commands map[CmdName]*Cmd
+ Commands map[string][]string
// DefaultCommand is the first command in the Upfile.
- DefaultCommand CmdName
-
- // DefaultEnvironment is the first inventory in the Upfile.
- DefaultEnvironment string
-
- lex *lexer
- text string
- indented bool
-}
-
-// Cmd to run conditionally if the conditions listed in ExecIf all exit with
-// zero.
-type Cmd struct {
- // ExecIfs any of the following commands exit with non-zero codes.
- ExecIfs []CmdName
-
- // Execs these commands in order using the default shell.
- Execs []string
-}
+ DefaultCommand string
-func ParseUpfile(rdr io.Reader) (*Config, error) {
- byt, err := ioutil.ReadAll(rdr)
- if err != nil {
- return nil, fmt.Errorf("read all: %w", err)
- }
- return parseUpfile(string(byt))
+ // Vars defined in the config file.
+ Vars map[string]string
}