~mna/straw

5674e42225afce2131283b8016162997a96f8a58 — Martin Angers 3 years ago ea73dc1 + a3011cd
Merge branch 'master' of git.sr.ht:~mna/straw
145 files changed, 3287 insertions(+), 532 deletions(-)

M README.md
A cmd/straw/args.go
A cmd/straw/args_test.go
M cmd/straw/straw.go
M cmd/straw/straw_test.go
A cmd/straw/testdata/config/straw.toml
A cmd/straw/testdata/config/x/.gitkeep
A cmd/straw/testdata/config/y/.gitkeep
A cmd/straw/testdata/configstages/straw.toml
A cmd/straw/testdata/configstages/x/.gitkeep
A cmd/straw/testdata/configstages/y/.gitkeep
A cmd/straw/testdata/configtest/straw.toml
A cmd/straw/testdata/configtest/straw_test.toml
A cmd/straw/testdata/configtest/x/.gitkeep
A cmd/straw/testdata/configtest/y/.gitkeep
M doc/DESIGN.md
A doc/app.toml
A doc/auth.toml
A doc/event.toml
M pkg/errors/errors.go
M pkg/exec/exec.go
M pkg/hook/core/core.go
M pkg/hook/core/core_test.go
A pkg/hook/tfhook/testdata/invalid/src/terraform/noconfig.tf
A pkg/hook/tfhook/testdata/invalid/src/terraform/straw.config/.gitkeep
A pkg/hook/tfhook/testdata/none/.gitkeep
A pkg/hook/tfhook/testdata/perstage/src/terraform/straw.toml
A pkg/hook/tfhook/testdata/perstage/src/terraform/test/file.tf
A pkg/hook/tfhook/testdata/template/src/terraform/file.tf.tpl
A pkg/hook/tfhook/testdata/template/src/terraform/straw.toml
A pkg/hook/tfhook/testdata/tmp/.gitignore
A pkg/hook/tfhook/testdata/valid/src/terraform/output.tf
A pkg/hook/tfhook/testdata/valid/src/terraform/straw.toml
A pkg/hook/tfhook/tfhook.go
A pkg/hook/tfhook/tfhook_test.go
M pkg/integration_test.go
M pkg/internal/testing/testing.go
M pkg/service/api/config.go
M pkg/service/api/resource.go
M pkg/service/api/service.go
M pkg/service/api/service_test.go
M pkg/service/api/testdata/src/apiconfig/straw.toml
A pkg/service/auth/config.go
A pkg/service/auth/resource.go
A pkg/service/auth/service.go
A pkg/service/event/config.go
A pkg/service/event/config_test.go
A pkg/service/event/resource.go
A pkg/service/event/service.go
A pkg/service/event/service_test.go
A pkg/service/event/testdata/dist/.gitkeep
A pkg/service/event/testdata/src/event/straw.toml
A pkg/service/event/testdata/src/eventconfig/straw.toml
A pkg/service/event/testdata/src/notevent/.gitkeep
A pkg/service/event/testdata/src/prodonlyevent/straw_prod.toml
A pkg/service/event/testdata/src/testonlyevent/straw_test.toml
M pkg/service/fn/config.go
M pkg/service/fn/service.go
M pkg/service/fn/service_test.go
M pkg/service/fn/testdata/src/fnconfig/straw.toml
M pkg/service/fn/testdata/src/testonlyfunc/straw_test.toml
M pkg/service/internal/bucket/config.go
M pkg/service/queue/config.go
M pkg/service/queue/service.go
M pkg/service/queue/service_test.go
M pkg/service/queue/testdata/src/queue.fifo/straw.toml
M pkg/service/queue/testdata/src/testonlyqueue/straw_test.toml
M pkg/service/service.go
M pkg/service/service_test.go
M pkg/service/site/service.go
M pkg/service/site/service_test.go
M pkg/service/site/testdata/src/config.site.com/straw.toml
M pkg/service/site/testdata/src/testonlysite/straw_test.toml
M pkg/service/storage/service.go
M pkg/service/storage/service_test.go
M pkg/service/storage/testdata/src/bucketcontents/straw.toml
M pkg/service/storage/testdata/src/testonlybucket/straw_test.toml
M pkg/service/topic/config.go
M pkg/service/topic/service.go
M pkg/service/topic/service_test.go
M pkg/service/topic/testdata/src/topic/straw.toml
M pkg/template/internal/esctpl/gen_templates.go
M pkg/testdata/02-bucket-base-props/src/storage/a/straw.toml
A pkg/testdata/23-api-minimal/dist/.gitkeep
A pkg/testdata/23-api-minimal/golden.json
A pkg/testdata/23-api-minimal/src/api/a/base/one/straw.toml
A pkg/testdata/23-api-minimal/src/api/a/base/straw.toml
A pkg/testdata/23-api-minimal/src/api/a/base/two/straw.toml
A pkg/testdata/23-api-minimal/src/api/a/notapath/.gitkeep
A pkg/testdata/23-api-minimal/src/api/a/straw.toml
A pkg/testdata/23-api-minimal/tmp/.gitignore
A pkg/testdata/24-api-root-handler/dist/func/fn/fn.zip
A pkg/testdata/24-api-root-handler/golden.json
A pkg/testdata/24-api-root-handler/src/api/a/straw.toml
A pkg/testdata/24-api-root-handler/src/api/a/{proxy+}/straw.toml
A pkg/testdata/24-api-root-handler/src/func/fn/main.go
A pkg/testdata/24-api-root-handler/src/func/fn/straw.toml
A pkg/testdata/24-api-root-handler/tmp/.gitignore
A pkg/testdata/25-api-many-handlers/dist/func/fn1/fn1.zip
A pkg/testdata/25-api-many-handlers/dist/func/fn2/fn2.zip
A pkg/testdata/25-api-many-handlers/dist/func/fn3/fn3.zip
A pkg/testdata/25-api-many-handlers/golden.json
A pkg/testdata/25-api-many-handlers/src/api/a/base/one/straw.toml
A pkg/testdata/25-api-many-handlers/src/api/a/base/straw.toml
A pkg/testdata/25-api-many-handlers/src/api/a/base/two/straw.toml
A pkg/testdata/25-api-many-handlers/src/api/a/straw.toml
A pkg/testdata/25-api-many-handlers/src/func/fn1/main.go
A pkg/testdata/25-api-many-handlers/src/func/fn1/straw.toml
A pkg/testdata/25-api-many-handlers/src/func/fn2/main.go
A pkg/testdata/25-api-many-handlers/src/func/fn2/straw.toml
A pkg/testdata/25-api-many-handlers/src/func/fn3/main.go
A pkg/testdata/25-api-many-handlers/src/func/fn3/straw.toml
A pkg/testdata/25-api-many-handlers/tmp/.gitignore
A pkg/testdata/28-terraform-copy/dist/.gitkeep
A pkg/testdata/28-terraform-copy/golden.json
A pkg/testdata/28-terraform-copy/src/terraform/f1.tf
A pkg/testdata/28-terraform-copy/src/terraform/f2.tf
A pkg/testdata/28-terraform-copy/src/terraform/prod/f2.tf
A pkg/testdata/28-terraform-copy/src/terraform/straw.toml
A pkg/testdata/28-terraform-copy/src/terraform/test/f2.tf
A pkg/testdata/28-terraform-copy/tmp/.gitignore
A pkg/testdata/29-terraform-template/dist/.gitkeep
A pkg/testdata/29-terraform-template/golden.json
A pkg/testdata/29-terraform-template/src/terraform/resources.tf.tpl
A pkg/testdata/29-terraform-template/src/terraform/straw.toml
A pkg/testdata/29-terraform-template/src/topic/t1/straw.toml
A pkg/testdata/29-terraform-template/tmp/.gitignore
A pkg/testdata/30-event-schedule/dist/func/fn/fn.zip
A pkg/testdata/30-event-schedule/golden.json
A pkg/testdata/30-event-schedule/src/event/cron/straw.toml
A pkg/testdata/30-event-schedule/src/event/rate/straw.toml
A pkg/testdata/30-event-schedule/src/func/fn/main.go
A pkg/testdata/30-event-schedule/src/func/fn/straw.toml
A pkg/testdata/30-event-schedule/tmp/.gitignore
M straw.go
M straw_test.go
M templates/api.tf.tpl
A templates/event.tf.tpl
A testdata/isdir/straw.toml/.gitkeep
A testdata/isprodstraw/straw_prod.toml
A testdata/isstraw/straw.toml
A testdata/isteststraw/straw_test.toml
A testdata/notadir
A testdata/notstraw/.gitkeep
M wiki
M README.md => README.md +64 -36
@@ 9,24 9,28 @@ The `straw` command generates an AWS "serverless"-type infrastructure from the p

## Description

Using convention and minimal configuration, the `straw` command generates [terraform][tf]
configuration files to build an AWS "serverless"-type infrastructure. The resources created
are isolated per "stage", so that exact replicas can be built e.g. for "test", "staging" or
"prod". Each stage is fully isolated.
Using a convention-based directory layout and minimal configuration, the `straw` command
generates [terraform][tf] configuration files to build an AWS "serverless"-type infrastructure.
The resources created are isolated per "stage", so that exact replicas can be built e.g. for
"test", "staging" or "prod".

What we mean by "serverless" is that the services used do not require explicit management
of servers or manual intervention to scale up or down. The following resources are
currently supported:

* `api`: AWS API Gateway APIs.
* `event`: AWS CloudWatch events.
* `func`: AWS Lambda functions.
* `queue`: AWS SQS queues.
* `topic`: AWS SNS topics.
* `storage`: AWS S3 buckets.
* `site`: AWS S3-hosted websites.
* `storage`: AWS S3 buckets.
* `terraform`: Custom terraform files.
* `topic`: AWS SNS topics.

Links between those resources can be created with a little configuration - e.g. to use
a function as handler for a queue, or as a subscriber for a topic, etc. Configuration is
done in a [TOML][toml] file named `straw.toml` in the resource's directory.
a function as handler for a queue, an API method handler, a subscriber for a topic, etc.
Configuration is done in a [TOML][toml] file named `straw.toml` in the resource's directory,
and `straw` takes care of generating the required permissions.

## Install



@@ 38,31 42,36 @@ repository and build the command with the `go` compiler (or use `go get`).
```
usage: straw [OPTIONS] STAGE

Automatically generate Terraform configuration based on the
directory layout of the src/ sub-directory under the specified
-dir directory (which defaults to the current one).

  -app-name name
    	The name of the application. (default <root directory name>)
  -apply
    	Apply the generated configuration. (default true)
  -aws-provider-version version
    	Use this AWS provider version condition. (default "~> 1.43")
  -aws-region region
    	The aws region to use in the configuration. (default "us-east-1")
  -dir directory
    	Generate terraform configuration for this directory. (default ".")
  -h	Print usage message.
  -help

  -no-apply
    	Do not apply the generated configuration. (default false)
  -out directory
    	Write terraform configuration under this directory. (default "terraform")
  -skip-init
    	Skip terraform init. Implies -apply flag. (default false)
  -terraform-version version
    	Use this terraform version condition. (default "> 0.11.0")
The straw command generates terraform configuration from a project's
directory layout. Each directory under "src/" can be associated
with a serverless service and be used to generate resources (instances
of that service).

The following options are available:
  -app
      The name of the generated app. (default: basename of the directory)
  -a, -apply
      Run 'terraform apply' automatically on the generated configuration.
      (default: true)
  -aws-provider-version
      Version condition for the terraform AWS provider. (default: ~> 1.43)
  -dir
      Root directory of the project to generate. (default: current
      directory)
  -h, -help
      Display this help message and exit. (default: false)
  -A, -no-apply
      Do not run 'terraform apply' automatically on the generated
      configuration. (default: false)
  -I, -no-init
      Do not run 'terraform init' automatically before running
      'terraform apply'. (default: false)
  -out
      Output directory where terraform configuration files are
      generated, with the stage appended as sub-directory.
      (default: ./terraform)
  -terraform-version
      Version condition for the terraform command. (default: > 0.11.0)
```

By convention, a specific directory layout is expected. For example, the


@@ 71,22 80,37 @@ following layout:
```
* app
  - src
    - api
      - pet_store
        - pets
          - {id}
            * straw.toml
          * straw.toml
        * straw.tom
    - event
      - cronjob
        *straw.toml
    - func
      - hello
        * straw.toml
        * index.js
        * straw.toml
    - queue
      - queue1
        * straw.toml
    - site
      - example.com
        * straw.toml
        * index.pug
        * straw.toml
    - storage
      - docsbucket
        * straw.toml
      - miscbucket
        * straw.toml
    - terraform
      - prod
        * prod_specific.tf
      * general_config.tf
      * straw.toml
    - topic
      - appnotif
        * straw.toml


@@ 103,11 127,13 @@ The following directories are recognized:

* Both `src/` and `dist/` folders must exist under the app's root directory
* The `src/` directory defines the resources to generate:
  - `func`, `queue`, `site`, `storage` and `topic` sub-directories are recognized and generate the expected resources.
  - `api`, `event`, `func`, `queue`, `site`, `storage` and `topic` sub-directories are recognized and generate the expected resources.
  - Under each of those directories, all sub-directories are distinct instances of this resource, provided they contain:
    - a (possibly empty) `straw.toml` file
    - a (possibly empty) `straw_<stage>.toml` file, where `<stage>` exactly matches the current stage
    - both `straw.toml` and `straw_<stage>.toml` files, where `straw_<stage>.toml` overrides configuration set by `straw.toml`
  - For `api`, directories under the API resource represent path parts (if they have a relevant straw configuration file).
  - Terraform and template files under `terraform` are copied over to the output stage directory (template files are executed with the list of resources as data).
* Under `dist/`, the build system must package the functions in a zip file as expected by AWS Lambda
  - Any tool can be used to write the function, as long as it generates a Node or Go zipped package
* Under `dist/`, the build system must generate the static files for the `site`


@@ 115,6 141,8 @@ The following directories are recognized:

## Changelog

* `v0.2.0` : refactor into maintainable code with Service, Resource, Validator, Generator and
Hook abstractions, add API Gateway support with Lambda handlers.
* `v0.1.0` : proof-of-concept, `func`s, `queue`s, `topic`s and `storage` (and `site`) are
supported, along with dependencies (queue handlers, topic subscribers, bucket notifications,
function dead letters).

A cmd/straw/args.go => cmd/straw/args.go +238 -0
@@ 0,0 1,238 @@
package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"

	"git.sr.ht/~mna/straw"
	"github.com/BurntSushi/toml"
)

// flags defines the command-line flags and arguments
// and holds the parsed values.
type flags struct {
	// non-flag arguments
	args []string
	// set of flags that were assigned on the command-line
	flags map[string]bool

	// NOTE: usage message must be changed if flags are modified.
	App                string `flag:"app"`
	Apply              bool   `flag:"a,apply"`
	AWSProviderVersion string `flag:"aws-provider-version"`
	Dir                string `flag:"dir"`
	DistDirName        string `flag:"dist-dir-name"`
	Help               bool   `flag:"h,help"`
	NoApply            bool   `flag:"A,no-apply"`
	NoInit             bool   `flag:"I,no-init"`
	Out                string `flag:"out"`
	SrcDirName         string `flag:"src-dir-name"`
	TerraformVersion   string `flag:"terraform-version"`
}

func newFlags() (*flags, error) {
	cur, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	return &flags{
		App: filepath.Base(cur),
		// apply defaults to true, but initialize to false because
		// if not set and no-apply is set, then it is no-apply.
		Apply:              false,
		AWSProviderVersion: defaultAWSProviderVersion,
		Dir:                cur,
		DistDirName:        defaultDistDirName,
		Help:               false,
		NoApply:            false,
		NoInit:             false,
		Out:                filepath.Join(cur, defaultOutputDir),
		SrcDirName:         defaultSrcDirName,
		TerraformVersion:   defaultTerraformVersion,
	}, nil
}

func (fl *flags) setArgs(args []string) {
	fl.args = args
}

func (fl *flags) setFlags(set map[string]bool) {
	fl.flags = set
}

func (fl *flags) validatePreConfig() error {
	// validate stage
	switch len(fl.args) {
	case 1:
		// ok
	case 0:
		return errors.New("expected a STAGE argument")
	default:
		return errors.New("expected a single STAGE argument")
	}
	// validate root directory
	if fl.Dir == "" {
		return errors.New("flag needs a non-empty value: -dir")
	}
	return nil
}

func (fl *flags) validatePostConfig() error {
	if fl.Apply && fl.NoApply {
		return errors.New("only one of -apply and -no-apply can be set")
	}
	requiredArgs := map[string]string{
		"app":           fl.App,
		"out":           fl.Out,
		"src-dir-name":  fl.SrcDirName,
		"dist-dir-name": fl.DistDirName,
	}
	for nm, val := range requiredArgs {
		if val == "" {
			return fmt.Errorf("flag needs a non-empty value: -%s", nm)
		}
	}
	return nil
}

func (fl *flags) applyConfig(cf straw.ConfigFiles) error {
	type config struct {
		Name               string   `toml:"name"`
		Apply              bool     `toml:"apply"`
		Init               bool     `toml:"init"`
		Out                string   `toml:"out"`
		SrcDirName         string   `toml:"src_dir_name"`
		DistDirName        string   `toml:"dist_dir_name"`
		AWSProviderVersion string   `toml:"aws_provider_version"`
		TerraformVersion   string   `toml:"terraform_version"`
		Stages             []string `toml:"stages"`
	}

	var validStages []string

	_, err := cf.EachIn(fl.Dir, func(file string) error {
		var c config
		meta, err := toml.DecodeFile(file, &c)
		if err != nil {
			return err
		}

		if !fl.flags["app"] && meta.IsDefined("name") {
			fl.App = c.Name
		}
		if !fl.flags["apply"] &&
			!fl.flags["no-apply"] &&
			!fl.flags["a"] &&
			!fl.flags["A"] &&
			meta.IsDefined("apply") {
			fl.Apply = c.Apply
			fl.NoApply = !c.Apply
		}
		if !fl.flags["no-init"] &&
			!fl.flags["I"] &&
			meta.IsDefined("init") {
			fl.NoInit = !c.Init
		}
		if !fl.flags["out"] && meta.IsDefined("out") {
			fl.Out = c.Out
		}
		if !fl.flags["src-dir-name"] && meta.IsDefined("src_dir_name") {
			fl.SrcDirName = c.SrcDirName
		}
		if !fl.flags["dist-dir-name"] && meta.IsDefined("dist_dir_name") {
			fl.DistDirName = c.DistDirName
		}
		if !fl.flags["aws-provider-version"] && meta.IsDefined("aws_provider_version") {
			fl.AWSProviderVersion = c.AWSProviderVersion
		}
		if !fl.flags["terraform-version"] && meta.IsDefined("terraform_version") {
			fl.TerraformVersion = c.TerraformVersion
		}
		if meta.IsDefined("stages") {
			validStages = c.Stages
		}
		return nil
	})

	if err != nil {
		return err
	}

	if len(validStages) > 0 {
		stage := fl.args[0]

		var isValid bool
		for _, vs := range validStages {
			if stage == vs {
				isValid = true
				break
			}
		}
		if !isValid {
			return fmt.Errorf("stage %s not in the list of valid stages: %v", stage, validStages)
		}
	}
	return nil
}

// parses args into v, using struct tags to detect flags.
// v must be a pointer to a struct. If v has a setArgs
// method, it is called with the list of non-flag arguments.
// If v has a setFlags method, it is called with the set of
// flags that were set by args.
func parseFlags(args []string, v interface{}) error {
	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.SetOutput(ioutil.Discard)
	fs.Usage = nil

	val := reflect.ValueOf(v).Elem()
	str := reflect.TypeOf(v).Elem()
	count := val.NumField()
	for i := 0; i < count; i++ {
		fld := val.Field(i)
		typ := str.Field(i)
		names := strings.Split(typ.Tag.Get("flag"), ",")

		for _, nm := range names {
			if nm == "" {
				continue
			}
			switch fld.Kind() {
			case reflect.Bool:
				fs.BoolVar(fld.Addr().Interface().(*bool), nm, fld.Bool(), "")
			case reflect.String:
				fs.StringVar(fld.Addr().Interface().(*string), nm, fld.String(), "")
			case reflect.Int:
				fs.IntVar(fld.Addr().Interface().(*int), nm, int(fld.Int()), "")
			default:
				panic(fmt.Sprintf("unsupported flag field kind: %s", fld.Kind()))
			}
		}
	}

	if err := fs.Parse(args); err != nil {
		return err
	}

	if sa, ok := v.(interface{ setArgs([]string) }); ok {
		sa.setArgs(fs.Args())
	}
	if sf, ok := v.(interface{ setFlags(map[string]bool) }); ok {
		set := make(map[string]bool)
		fs.Visit(func(fl *flag.Flag) {
			set[fl.Name] = true
		})
		if len(set) > 0 {
			sf.setFlags(set)
		}
	}

	return nil
}

A cmd/straw/args_test.go => cmd/straw/args_test.go +112 -0
@@ 0,0 1,112 @@
package main

import (
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"
)

func TestParseFlags(t *testing.T) {
	cur, err := os.Getwd()
	require.NoError(t, err)
	out, err := filepath.Abs(filepath.Join(cur, defaultOutputDir))
	require.NoError(t, err)

	cases := []struct {
		args []string
		want flags
	}{
		{
			nil,
			flags{
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				DistDirName:        defaultDistDirName,
				Out:                out,
				SrcDirName:         defaultSrcDirName,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"toto"},
			flags{
				args:               []string{"toto"},
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				DistDirName:        defaultDistDirName,
				Out:                out,
				SrcDirName:         defaultSrcDirName,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"-h"},
			flags{
				args:               []string{},
				flags:              map[string]bool{"h": true},
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				DistDirName:        defaultDistDirName,
				Help:               true,
				Out:                out,
				SrcDirName:         defaultSrcDirName,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"-app", "test", "-A", "-no-init"},
			flags{
				args:               []string{},
				flags:              map[string]bool{"app": true, "A": true, "no-init": true},
				App:                "test",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				DistDirName:        defaultDistDirName,
				NoApply:            true,
				NoInit:             true,
				Out:                out,
				SrcDirName:         defaultSrcDirName,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"-src-dir-name", "x", "-dist-dir-name", "y"},
			flags{
				args:               []string{},
				flags:              map[string]bool{"src-dir-name": true, "dist-dir-name": true},
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				DistDirName:        "y",
				Out:                out,
				SrcDirName:         "x",
				TerraformVersion:   defaultTerraformVersion,
			},
		},
	}

	for _, c := range cases {
		t.Run(strings.Join(c.args, " "), func(t *testing.T) {
			fl, err := newFlags()
			require.NoError(t, err)
			err = parseFlags(c.args, fl)
			require.NoError(t, err)

			require.Equal(t, c.want, *fl)
		})
	}
}

func TestParseArgsError(t *testing.T) {
	fl, err := newFlags()
	require.NoError(t, err)
	err = parseFlags([]string{"-zz"}, fl)
	require.Error(t, err)
	require.Contains(t, err.Error(), "-zz")
}

M cmd/straw/straw.go => cmd/straw/straw.go +85 -104
@@ 2,19 2,17 @@ package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/exec"
	"git.sr.ht/~mna/straw/pkg/hook/core"
	"git.sr.ht/~mna/straw/pkg/hook/tfhook"
	"git.sr.ht/~mna/straw/pkg/service/api"
	"git.sr.ht/~mna/straw/pkg/service/event"
	"git.sr.ht/~mna/straw/pkg/service/fn"
	"git.sr.ht/~mna/straw/pkg/service/queue"
	"git.sr.ht/~mna/straw/pkg/service/site"


@@ 28,31 26,26 @@ const (
	defaultTerraformVersion   = "> 0.11.0"
	defaultAWSProviderVersion = "~> 1.43"
	defaultOutputDir          = "terraform"
	defaultSrcDirName         = "src"
	defaultDistDirName        = "dist"
	bootstrapDirSuffix        = "-bootstrap"
	userGuideAddress          = "https://man.sr.ht/~mna/straw/"
)

// flags defines the command-line flags and arguments
// and holds the parsed values.
type flags struct {
	Args []string // non-flag arguments

	App                string `flag:"app"`
	Apply              bool   `flag:"a,apply"`
	AWSProviderVersion string `flag:"aws-provider-version"`
	Dir                string `flag:"dir"`
	Help               bool   `flag:"h,help"`
	NoApply            bool   `flag:"A,no-apply"`
	NoInit             bool   `flag:"I,no-init"`
	Out                string `flag:"out"`
	TerraformVersion   string `flag:"terraform-version"`
}

// messager is the interface implemented by errors that
// have user-targeted messages to display.
type messager interface {
	Message() string
}

// successer is the interface implemented by errors that
// are used to propagate a message to display, but other
// than that the command should exit with a success code
// if Success returns true.
type successer interface {
	Success() bool
}

// osenv provides the OS-specific environment, such
// as standard output and input and exit call.
type osenv struct {


@@ 82,21 75,28 @@ func (o osenv) usage() {
	const msg = `usage: straw [OPTIONS] STAGE

The straw command generates terraform configuration from a project's
directory layout. Each directory under "src/" can be associated
with a serverless service and be used to generate resources (instances
of that service).
directory layout. Each subdirectory under 'ROOTDIR/SRCNAME' can be
associated with a serverless service and be used to generate resources
(distinct instances of that service).

The generated infrastructure is completely isolated per STAGE, and
resources can be further configured via simple TOML files. See the
user guide at %s for more information.

The following options are available:
  -app
  -app APPNAME
      The name of the generated app. (default: basename of the directory)
  -a, -apply
      Run 'terraform apply' automatically on the generated configuration.
      (default: true)
  -aws-provider-version
  -aws-provider-version AWSCONDITION
      Version condition for the terraform AWS provider. (default: %s)
  -dir
  -dir ROOTDIR
      Root directory of the project to generate. (default: current
      directory)
  -dist-dir-name DISTNAME
      Name of the 'dist' subdirectory under the root directory.
      (default: dist)
  -h, -help
      Display this help message and exit. (default: false)
  -A, -no-apply


@@ 105,15 105,23 @@ The following options are available:
  -I, -no-init
      Do not run 'terraform init' automatically before running
      'terraform apply'. (default: false)
  -out
  -out OUTDIR
      Output directory where terraform configuration files are
      generated, with the stage appended as sub-directory.
      (default: ./%s)
  -terraform-version
  -src-dir-name SRCNAME
      Name of the 'src' subdirectory under the root directory.
      (default: src)
  -terraform-version TFCONDITION
      Version condition for the terraform command. (default: %s)

`
	fmt.Fprintf(o.Stdout, msg, defaultAWSProviderVersion, defaultOutputDir, defaultTerraformVersion)
	fmt.Fprintf(o.Stdout, msg,
		userGuideAddress,
		defaultAWSProviderVersion,
		defaultOutputDir,
		defaultTerraformVersion,
	)
}

func main() {


@@ 123,33 131,39 @@ func main() {
		Stdin:  os.Stdin,
		Exit:   os.Exit,
	}
	run(oe, os.Args[1:])
}

	fl, err := parseArgs(os.Args[1:])
func run(oe osenv, args []string) {
	fl, err := newFlags()
	if err != nil {
		oe.exit(err, 1)
		return
	}
	if err := parseFlags(args, fl); err != nil {
		oe.exitWithUsage(err, 2)
		return
	}
	run(oe, fl)
}

func run(oe osenv, fl *flags) {
	if fl.Help {
		oe.usage()
		return
	}

	switch len(fl.Args) {
	case 1:
		// ok
	case 0:
		oe.exitWithUsage(errors.New("expected a STAGE argument"), 2)
	if err := fl.validatePreConfig(); err != nil {
		oe.exitWithUsage(err, 2)
		return
	default:
		oe.exitWithUsage(errors.New("expected a single STAGE argument"), 2)
	}

	stage := fl.args[0]
	cf := straw.NewConfigFiles(stage)
	if err := fl.applyConfig(cf); err != nil {
		oe.exit(fmt.Errorf("failed to apply configuration file: %s", err), 1)
		return
	}
	if fl.Apply && fl.NoApply {
		oe.exitWithUsage(errors.New("only one of -apply and -no-apply can be set"), 2)

	if err := fl.validatePostConfig(); err != nil {
		oe.exitWithUsage(err, 2)
		return
	}



@@ 159,25 173,29 @@ func run(oe osenv, fl *flags) {
		Stderr: oe.Stderr,
	}

	stage := fl.Args[0]
	rootDir, _ := filepath.Abs(fl.Dir)
	stageDir, _ := filepath.Abs(filepath.Join(fl.Out, stage))
	bootstrapDir := stageDir + bootstrapDirSuffix

	rc := straw.RunConfig{
		Stage:   stage,
		AppName: fl.App,
		Stage:       stage,
		AppName:     fl.App,
		ConfigFiles: cf,

		RootDir:               rootDir,
		SrcDirName:            fl.SrcDirName,
		DistDirName:           fl.DistDirName,
		TerraformStageDir:     stageDir,
		TerraformBootstrapDir: bootstrapDir,

		RunTerraformApply: (fl.Apply || !fl.NoApply),
		RunTerraformInit:  !fl.NoInit,
		RunTerraformApply:       (fl.Apply || !fl.NoApply),
		RunTerraformInit:        !fl.NoInit,
		TerraformApplyRequested: (fl.Apply || !fl.NoApply),

		TerraformVersionCondition:            fl.TerraformVersion,
		TerraformAWSProviderVersionCondition: fl.AWSProviderVersion,
	}

	if !tfcmd.Available() {
		rc.RunTerraformApply = false
		tfcmd = nil


@@ 197,8 215,22 @@ func run(oe osenv, fl *flags) {
			TF:       tfcmd,
			Progress: os.Stdout,
		},
		&tfhook.Hook{
			RC: rc,
			CR: cr,
		},
	}
	apiSvc := &api.Service{
		RC: rc,
		CR: cr,
	}
	services := []straw.Service{
		apiSvc,
		apiSvc.PathMethod(),
		&event.Service{
			RC: rc,
			CR: cr,
		},
		&fn.Service{
			RC: rc,
			CR: cr,


@@ 239,65 271,14 @@ func run(oe osenv, fl *flags) {
	}
	if err := cmd.Run(); err != nil {
		if em, ok := err.(messager); ok {
			oe.exitWithMsg(em, 4)
			exitCode := 4
			if sc, ok := err.(successer); ok && sc.Success() {
				exitCode = 0
			}
			oe.exitWithMsg(em, exitCode)
			return
		}
		oe.exit(err, 3)
		return
	}
}

func parseArgs(args []string) (*flags, error) {
	cur, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	fl := &flags{
		App:                filepath.Base(cur),
		Apply:              false,
		AWSProviderVersion: defaultAWSProviderVersion,
		Dir:                cur,
		Help:               false,
		NoApply:            false,
		NoInit:             false,
		Out:                filepath.Join(cur, defaultOutputDir),
		TerraformVersion:   defaultTerraformVersion,
	}

	fs := flag.NewFlagSet("", flag.ContinueOnError)
	fs.SetOutput(ioutil.Discard)
	fs.Usage = nil

	val := reflect.ValueOf(fl).Elem()
	str := reflect.TypeOf(fl).Elem()
	count := val.NumField()
	for i := 0; i < count; i++ {
		fld := val.Field(i)
		typ := str.Field(i)
		names := strings.Split(typ.Tag.Get("flag"), ",")

		for _, nm := range names {
			if nm == "" {
				continue
			}
			switch fld.Kind() {
			case reflect.Bool:
				fs.BoolVar(fld.Addr().Interface().(*bool), nm, fld.Bool(), "")
			case reflect.String:
				fs.StringVar(fld.Addr().Interface().(*string), nm, fld.String(), "")
			case reflect.Int:
				fs.IntVar(fld.Addr().Interface().(*int), nm, int(fld.Int()), "")
			default:
				panic(fmt.Sprintf("unsupported flag field kind: %s", fld.Kind()))
			}
		}
	}

	if err := fs.Parse(args); err != nil {
		return nil, err
	}
	fl.Args = fs.Args()

	return fl, nil
}

M cmd/straw/straw_test.go => cmd/straw/straw_test.go +68 -107
@@ 2,7 2,7 @@ package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"


@@ 17,76 17,103 @@ func TestRun(t *testing.T) {
	os.Setenv("AWS_DEFAULT_REGION", "")

	cases := []struct {
		fl       flags
		args     string
		region   string
		wantCode int
		wantOut  string
		wantErr  string
	}{
		{
			fl: flags{
				Help: true,
			},
			args:     "-h",
			wantCode: 0,
			wantOut:  "usage:",
		},
		{
			fl:       flags{},
			args:     "",
			wantCode: 2,
			wantErr:  "expected a STAGE argument",
			wantOut:  "usage:",
		},
		{
			fl:       flags{Args: []string{"A", "B"}},
			args:     "A B",
			wantCode: 2,
			wantErr:  "expected a single STAGE argument",
			wantOut:  "usage:",
		},
		{
			fl: flags{
				Args:    []string{"A"},
				Apply:   true,
				NoApply: true,
			},
			args:     "-apply -A A",
			wantCode: 2,
			wantErr:  "only one of -apply and -no-apply",
			wantOut:  "usage:",
		},
		{
			fl: flags{
				Args:    []string{"test"},
				NoApply: true,
				Dir:     "testdata/invalid",
				Out:     "testdata/tmp",
			},
			args:     "-app test -no-apply -dir testdata/invalid -out testdata/tmp test",
			wantCode: 3,
			wantErr:  "no such file or directory",
		},
		{
			fl: flags{
				Args:    []string{"test"},
				NoApply: true,
				Dir:     "testdata/valid",
				Out:     "testdata/tmp",
			},
			args:     "-app test -A -dir testdata/valid -out testdata/tmp test",
			wantCode: 3,
			wantErr:  "no AWS region set",
		},
		{
			fl: flags{
				Args:    []string{"test"},
				NoApply: true,
				Dir:     "testdata/valid",
				Out:     "testdata/tmp",
			},
			args:     "-app test -A -dir testdata/valid -out testdata/tmp test",
			region:   "us-east-1",
			wantCode: 4,
			wantErr:  "remote state storage is not setup",
		},
		{
			args:     "-app '' -A -dir testdata/valid -out testdata/tmp test",
			wantCode: 2,
			wantErr:  "non-empty value: -app",
			wantOut:  "usage:",
		},
		{
			args:     "-app test -A -dir '' -out testdata/tmp test",
			wantCode: 2,
			wantErr:  "non-empty value: -dir",
			wantOut:  "usage:",
		},
		{
			args:     "-app test -A -dir testdata/config -out testdata/tmp test",
			region:   "us-east-1",
			wantCode: 4,
			wantErr:  "remote state storage is not setup",
		},
		{
			args:     "-app test -src-dir-name src -A -dir testdata/config -out testdata/tmp test",
			wantCode: 3,
			wantErr:  "no such file or directory",
		},
		{
			args:     "-dir testdata/configtest test",
			region:   "us-east-1",
			wantCode: 4,
			wantErr:  "remote state storage is not setup",
		},
		{
			args:     "-app test -dir testdata/configstages -out testdata/tmp test",
			region:   "us-east-1",
			wantCode: 1,
			wantErr:  "stage test not in the list",
		},
	}

	for _, c := range cases {
		t.Run(fmt.Sprintf("%#v", c.fl), func(t *testing.T) {
		t.Run(c.args, func(t *testing.T) {
			defer func() {
				path := filepath.Join("testdata", "tmp")
				dirs, err := ioutil.ReadDir(path)
				require.NoError(t, err)
				for _, dir := range dirs {
					if !dir.IsDir() {
						continue
					}
					err = os.RemoveAll(filepath.Join(path, dir.Name()))
					require.NoError(t, err)
				}
			}()

			var code int
			var out, err bytes.Buffer
			oe := osenv{


@@ 100,9 127,17 @@ func TestRun(t *testing.T) {
				defer os.Setenv("AWS_REGION", "")
			}

			run(oe, &c.fl)
			args := strings.Fields(c.args)
			// special-case: if a value is "" or '', translate to
			// empty string
			for i := 0; i < len(args); i++ {
				if args[i] == "''" || args[i] == `""` {
					args[i] = ""
				}
			}
			run(oe, args)

			require.Equal(t, c.wantCode, code)
			assert.Equal(t, c.wantCode, code)

			if c.wantOut == "" {
				require.Empty(t, out.String(), "stdout")


@@ 118,77 153,3 @@ func TestRun(t *testing.T) {
		})
	}
}

func TestParseArgs(t *testing.T) {
	cur, err := os.Getwd()
	require.NoError(t, err)
	out, err := filepath.Abs(filepath.Join(cur, defaultOutputDir))
	require.NoError(t, err)

	cases := []struct {
		args []string
		want flags
	}{
		{
			nil,
			flags{
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				Out:                out,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"toto"},
			flags{
				Args:               []string{"toto"},
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				Out:                out,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"-h"},
			flags{
				Args:               []string{},
				App:                "straw",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				Help:               true,
				Out:                out,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
		{
			[]string{"-app", "test", "-A", "-no-init"},
			flags{
				Args:               []string{},
				App:                "test",
				AWSProviderVersion: defaultAWSProviderVersion,
				Dir:                cur,
				NoApply:            true,
				NoInit:             true,
				Out:                out,
				TerraformVersion:   defaultTerraformVersion,
			},
		},
	}

	for _, c := range cases {
		t.Run(strings.Join(c.args, " "), func(t *testing.T) {
			got, err := parseArgs(c.args)
			require.NoError(t, err)

			assert.Equal(t, c.want, *got)
		})
	}
}

func TestParseArgsError(t *testing.T) {
	_, err := parseArgs([]string{"-zz"})
	require.Error(t, err)
	require.Contains(t, err.Error(), "-zz")
}

A cmd/straw/testdata/config/straw.toml => cmd/straw/testdata/config/straw.toml +2 -0
@@ 0,0 1,2 @@
src_dir_name = "x"
dist_dir_name = "y"

A cmd/straw/testdata/config/x/.gitkeep => cmd/straw/testdata/config/x/.gitkeep +0 -0
A cmd/straw/testdata/config/y/.gitkeep => cmd/straw/testdata/config/y/.gitkeep +0 -0
A cmd/straw/testdata/configstages/straw.toml => cmd/straw/testdata/configstages/straw.toml +4 -0
@@ 0,0 1,4 @@
stages = ["prod", "dev"]
src_dir_name = "x"
dist_dir_name = "y"
apply = false

A cmd/straw/testdata/configstages/x/.gitkeep => cmd/straw/testdata/configstages/x/.gitkeep +0 -0
A cmd/straw/testdata/configstages/y/.gitkeep => cmd/straw/testdata/configstages/y/.gitkeep +0 -0
A cmd/straw/testdata/configtest/straw.toml => cmd/straw/testdata/configtest/straw.toml +4 -0
@@ 0,0 1,4 @@
name = "config"
apply = false
init = false
stages = ["test", "prod"]

A cmd/straw/testdata/configtest/straw_test.toml => cmd/straw/testdata/configtest/straw_test.toml +5 -0
@@ 0,0 1,5 @@
out = "testdata/tmp"
src_dir_name = "x"
dist_dir_name = "y"
aws_provider_version = "~> 1.43"
terraform_version = "> 0.1.0"

A cmd/straw/testdata/configtest/x/.gitkeep => cmd/straw/testdata/configtest/x/.gitkeep +0 -0
A cmd/straw/testdata/configtest/y/.gitkeep => cmd/straw/testdata/configtest/y/.gitkeep +0 -0
M doc/DESIGN.md => doc/DESIGN.md +8 -13
@@ 12,20 12,15 @@ Details of `src`:
* ✅ `src/queue/...` : maps to an SQS queue. Path under `sqs` indicates the queue. Lambda funcs handle messages.
* ✅ `src/topic/...` : maps to an SNS topic. SQS queues or Lambda funcs can subscribe to topics.
* ✅ `src/func/...` : maps to Lambda functions. The zip package to deploy must be in the same directory name under `dist/func` and the zip file name is the base directory of the func with the .zip extension.
* ✅ `src/api/...` : maps to an API Gateway and associated Lambda.
* ✅ `src/terraform/...` : terraform files that get copied as-is to the stage's folder for additional customization and infrastructure.
* ✅ `src/event/...` : maps to a CloudWatch event cron/rate and associated Lambda, via the `straw.toml` file.
* ✅ `straw.toml` configuration file at the root directory to configure the project.

* 🚧 `src/api/...` : maps to an API Gateway and associated Lambda, and possibly Cognito authorizers. Can be configured to use a custom domain setup in `src/domain`.
  - Each managed directory under `api` is an API Gateway REST API
  - Each subdirectory of an API is an API Gateway Resource
  - Each resource defaults to ANY http method, use "resource:verb" to configure specific methods (e.g. `/api/website/blog:get`).
  - Path parameters are specified via directory names in curly braces, e.g. `/api/website/blog/{year}`.
  - Further configuration is via `straw.toml` or `straw_<stage>.toml` at each resource-directory level.
  - Default handler function to the api name + resource name?
* 🚧 `src/domain/...` : maps to custom domains in Route 53 and SSL certificates in ACM.
* 🚧 `src/auth/...` : maps to Cognito User Pools, with optional Lambdas for auth events.
* 🚧 `src/api/...` : support Cognito authorizers and custom domain setup in `src/domain`.

* `src/auth/...` : maps to Cognito User Pools, with optional Lambdas for auth events.
* `src/schedule/...` : maps to a CloudWatch event cron/rate and associated Lambda, via the `straw.toml` file.
* `src/terraform/...` : terraform files that get copied as-is to the stage's folder for additional customization and infrastructure.
  - Could still receive the `*config` as data?
* `src/domain/...` : maps to custom domains in Route 53 and SSL certificates in ACM.

Unclear how to handle:



@@ 38,5 33,5 @@ Unclear if needed:
* ❓ `src/role/...` : pre-defined IAM roles (by default, straw generates the roles as needed)?
* ❓ VPC configuration?

All AWS services should be completely isolated per stage (e.g. "development", "staging", "production").
All AWS services are completely isolated per stage (e.g. "development", "test", "staging", "production").


A doc/app.toml => doc/app.toml +13 -0
@@ 0,0 1,13 @@
name = "app"
apply = true
init = false
out = "output_directory"

src_dir_name = "src"
dist_dir_name = "dist"

aws_provider_version = "~> x"
terraform_version = "> 0.11.0"

stages = ["test", "staging", "prod"]


A doc/auth.toml => doc/auth.toml +62 -0
@@ 0,0 1,62 @@
# TODO: no SMS support for now
# TODO: no identity provider support for now
# TODO: no multiple clients support for now

name = "pool_name"
mfa = "on|off|optional"
alias_attributes = ["email"]
username_attributes = []
auto_verified_attributes = []

client_name = "defaults to app"

[verification_message]
  default_email_option = "CONFIRM_WITH_LINK"
  email_message = ""
  email_subject = ""

[admin]
  allow_admin_create_user_only = true
  unused_account_validity = "3d"
  invite_email_message = ""
  invite_email_subject = ""

[device]
  challenge_required_on_new_device = true
  device_only_remembered_on_user_prompt = true

[tags]
  x = "y"

[password_policy]
  minimum_length = 10
  require_lowercase = true
  require_numbers = true
  require_symbols = true
  require_uppercase = true

[triggers]
  create_auth_challenge = "fn_id"
  custom_message = "fn_id"
  define_auth_challenge = "fn_id"
  post_authentication = "fn_id"
  post_confirmation = "fn_id"
  pre_authentication = "fn_id"
  pre_sign_up = "fn_id"
  pre_token_generation = "fn_id"
  user_migration = "fn_id"
  verify_auth_challenge_response = "fn_id"

[[schema_attributes]]
  data_type = "Boolean|Number|String|DateTime"
  developer_only = false
  mutable = true
  name = ""
  required = true
  # for strings
  min_length = 3
  max_length = 10
  # for numbers
  min_value = 1
  max_value = 10


A doc/event.toml => doc/event.toml +31 -0
@@ 0,0 1,31 @@
id = "override id"
name = "override name"
is_prefix = true
description = "yep"

# optional, otherwise straw generates required permissions
role = "role arn already in AWS"

enabled = true

[schedule]
  type = "rate"
  value = "5h" # need to support "d" as valid suffix

# or...

[schedule]
  type = "cron"
  value = "minutes hours dom month dow year"

# or...
raw_event_pattern = '''
{
  "some": "json"
}
'''

[[targets]]
  target_id = "target id"
  type = "func|topic|queue|etc."
  id = "id of func|topic|etc."

M pkg/errors/errors.go => pkg/errors/errors.go +18 -4
@@ 20,7 20,7 @@ or, if using aws-vault to provide AWS credentials (recommended):

  $ (cd %[2]s && terraform init)

or, if using aws-vault to provide AWS credentials (recommended):
or, if using aws-vault to provide AWS credentials:

  $ (cd %[2]s && aws-vault exec $AWS_PROFILE -- terraform init)



@@ 28,7 28,7 @@ and apply with:

  $ (cd %[2]s && terraform apply)

or
or, using aws-vault:

  $ (cd %[2]s && aws-vault exec $AWS_PROFILE -- terraform apply)
`


@@ 75,8 75,9 @@ func NewStage(err error, stage, dir string) *ApplyError {
// ApplyError indicates that a terraform apply (and possibly
// init) is required to apply the configuration.
type ApplyError struct {
	Err error
	msg string
	Err     error
	success bool
	msg     string
}

// Error returns the error message for an ApplyError.


@@ 89,3 90,16 @@ func (e *ApplyError) Error() string {
func (e *ApplyError) Message() string {
	return e.msg
}

// MarkSuccess marks the "error" as a success, i.e. the
// error is returned for its message to be displayed, but the
// command should exit with a success.
func (e *ApplyError) MarkSuccess() {
	e.success = true
}

// Success returns true if the error was only returned for its
// message to display, but the command should exit with success.
func (e *ApplyError) Success() bool {
	return e.success
}

M pkg/exec/exec.go => pkg/exec/exec.go +16 -14
@@ 12,11 12,6 @@ import (
	"git.sr.ht/~mna/straw/pkg/service"
)

const (
	srcDir  = "src"
	distDir = "dist"
)

// Cmd runs the straw command based on the configuration.
type Cmd struct {
	RC straw.RunConfig


@@ 60,8 55,8 @@ func (c *Cmd) Run() error {

	// scan src dir
	fmt.Fprintln(c.Progress, "==> scanning src directory...")
	src := filepath.Join(c.RC.RootDir, srcDir)
	dist := filepath.Join(c.RC.RootDir, distDir)
	src := filepath.Join(c.RC.RootDir, c.RC.SrcDirName)
	dist := filepath.Join(c.RC.RootDir, c.RC.DistDirName)
	dirs, err := ioutil.ReadDir(src)
	if err != nil {
		return err


@@ 97,7 92,12 @@ func (c *Cmd) Run() error {
	for _, r := range c.CR.Resources {
		if v, ok := r.(straw.Validator); ok {
			if err := v.Validate(c.Registries); err != nil {
				return err
				typ, id := r.TypeID()
				return &errors.ResourceError{
					Err:  err,
					Type: typ,
					ID:   id,
				}
			}
		}
	}


@@ 133,13 133,18 @@ func (c *Cmd) Run() error {
		return c.TF.Exec("apply", c.RC.TerraformStageDir)
	}

	return errors.NewStage(straw.ErrApplyRequired, c.RC.Stage, c.RC.TerraformStageDir)
	ae := errors.NewStage(straw.ErrApplyRequired, c.RC.Stage, c.RC.TerraformStageDir)
	// treat this error as success (exit code 0) if apply was not requested.
	if !c.RC.TerraformApplyRequested {
		ae.MarkSuccess()
	}
	return ae
}

func validateRunConfig(rc straw.RunConfig) error {
	// source directory must have a "src/" subdirectory
	// and a "dist/" subdirectory
	for _, v := range []string{srcDir, distDir} {
	for _, v := range []string{rc.SrcDirName, rc.DistDirName} {
		p := filepath.Join(rc.RootDir, v)
		st, err := os.Stat(p)
		if err != nil {


@@ 149,8 154,5 @@ func validateRunConfig(rc straw.RunConfig) error {
			return fmt.Errorf("%s is not a directory", p)
		}
	}
	if err := service.ValidateStage(rc.Stage); err != nil {
		return err
	}
	return nil
	return service.ValidateStage(rc.Stage)
}

M pkg/hook/core/core.go => pkg/hook/core/core.go +1 -0
@@ 15,6 15,7 @@ import (
const tfStateFile = "terraform.tfstate"

var (
	_ straw.Hook      = (*Hook)(nil)
	_ straw.Generator = (*Hook)(nil)
	_ straw.Validator = (*Hook)(nil)
)

M pkg/hook/core/core_test.go => pkg/hook/core/core_test.go +9 -0
@@ 155,6 155,15 @@ func TestRun(t *testing.T) {
		t.Run(c.desc, func(t *testing.T) {
			c.ess.RC.TerraformStageDir = fmt.Sprintf("testdata/tmp/t%d", i)
			c.ess.RC.TerraformBootstrapDir = fmt.Sprintf("testdata/tmp/t%d-b", i)
			defer func() {
				if t.Failed() {
					return
				}
				err := os.RemoveAll(c.ess.RC.TerraformStageDir)
				require.NoError(t, err)
				err = os.RemoveAll(c.ess.RC.TerraformBootstrapDir)
				require.NoError(t, err)
			}()

			os.Setenv("AWS_REGION", c.region)
			h := newHook(c.ess)

A pkg/hook/tfhook/testdata/invalid/src/terraform/noconfig.tf => pkg/hook/tfhook/testdata/invalid/src/terraform/noconfig.tf +1 -0
@@ 0,0 1,1 @@


A pkg/hook/tfhook/testdata/invalid/src/terraform/straw.config/.gitkeep => pkg/hook/tfhook/testdata/invalid/src/terraform/straw.config/.gitkeep +0 -0
A pkg/hook/tfhook/testdata/none/.gitkeep => pkg/hook/tfhook/testdata/none/.gitkeep +0 -0
A pkg/hook/tfhook/testdata/perstage/src/terraform/straw.toml => pkg/hook/tfhook/testdata/perstage/src/terraform/straw.toml +0 -0
A pkg/hook/tfhook/testdata/perstage/src/terraform/test/file.tf => pkg/hook/tfhook/testdata/perstage/src/terraform/test/file.tf +1 -0
@@ 0,0 1,1 @@


A pkg/hook/tfhook/testdata/template/src/terraform/file.tf.tpl => pkg/hook/tfhook/testdata/template/src/terraform/file.tf.tpl +8 -0
@@ 0,0 1,8 @@
output "file" {
  value = "${var.environment}"
}
{{range .Resources}}
output "{{.Service.Type}}_{{.ID}}" {
  value = "{{.ID}}"
}
{{end}}

A pkg/hook/tfhook/testdata/template/src/terraform/straw.toml => pkg/hook/tfhook/testdata/template/src/terraform/straw.toml +0 -0
A pkg/hook/tfhook/testdata/tmp/.gitignore => pkg/hook/tfhook/testdata/tmp/.gitignore +2 -0
@@ 0,0 1,2 @@
*
!.gitignore

A pkg/hook/tfhook/testdata/valid/src/terraform/output.tf => pkg/hook/tfhook/testdata/valid/src/terraform/output.tf +3 -0
@@ 0,0 1,3 @@
output "test" {
  value = "output"
}

A pkg/hook/tfhook/testdata/valid/src/terraform/straw.toml => pkg/hook/tfhook/testdata/valid/src/terraform/straw.toml +0 -0
A pkg/hook/tfhook/tfhook.go => pkg/hook/tfhook/tfhook.go +171 -0
@@ 0,0 1,171 @@
package tfhook

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/template"
)

var (
	_ straw.Hook      = (*Hook)(nil)
	_ straw.Generator = (*Hook)(nil)
)

const terraformDirName = "terraform"

// Hook is the hook implementation for the installation of
// custom terraform files. It reads files under the source's
// "terraform" directory (if it has a relevant straw
// configuration file) and copies them over to the stage's
// output directory during generation.
type Hook struct {
	RC straw.RunConfig
	CR *straw.CollectedResources
}

// Run checks if there are custom terraform files to copy
// and if so, registers the Generator.
func (h *Hook) Run() error {
	tfsrc := filepath.Join(h.RC.RootDir, h.RC.SrcDirName, terraformDirName)
	stagesrc := filepath.Join(tfsrc, h.RC.Stage)

	checks := []struct {
		dir     string
		checkFn func(string) (bool, error)
	}{
		{tfsrc, h.RC.ConfigFiles.IsStrawDir},
		{stagesrc, dirExists},
	}
	for _, check := range checks {
		exists, err := check.checkFn(check.dir)
		if err != nil {
			return err
		}
		if !exists {
			return nil
		}

		dirs, err := ioutil.ReadDir(check.dir)
		if err != nil {
			return err
		}

		for _, dir := range dirs {
			if dir.IsDir() {
				continue
			}
			if isTerraformFile(dir.Name()) {
				h.CR.Generators = append(h.CR.Generators, h)
				return nil
			}
		}
	}

	return nil
}

// Generate copies the custom terraform files over to the target
// directory.
func (h *Hook) Generate(dir string, reg *straw.Registries) error {
	tfsrc := filepath.Join(h.RC.RootDir, h.RC.SrcDirName, terraformDirName)
	dirs, err := ioutil.ReadDir(tfsrc)
	if err != nil {
		return err
	}
	if err := h.generateDirs(reg, tfsrc, dir, dirs); err != nil {
		return err
	}

	stageSrc := filepath.Join(tfsrc, h.RC.Stage)
	exists, err := dirExists(stageSrc)
	if err != nil {
		return err
	}
	if !exists {
		return nil
	}

	dirs, err = ioutil.ReadDir(stageSrc)
	if err != nil {
		return err
	}
	if err := h.generateDirs(reg, stageSrc, dir, dirs); err != nil {
		return err
	}
	return nil
}

func (h *Hook) generateDirs(reg *straw.Registries, srcDir, dstDir string, dirs []os.FileInfo) error {
	for _, dir := range dirs {
		if dir.IsDir() {
			continue
		}

		if isTerraformFile(dir.Name()) {
			var err error

			switch {
			case strings.HasSuffix(dir.Name(), ".tf.tpl"):
				t := template.New(filepath.Join(srcDir, dir.Name())).
					Data("Resources", reg.Resources).
					ResourcesMap(reg.Resources).
					EnvMap(reg.Environment)
				err = t.Execute(dstDir)

			default:
				err = copyFile(
					filepath.Join(srcDir, dir.Name()),
					filepath.Join(dstDir, dir.Name()),
				)
			}

			if err != nil {
				return err
			}
		}
	}
	return nil
}

func isTerraformFile(name string) bool {
	ext := filepath.Ext(name)
	return ext == ".tf" || ext == ".tfvars" || strings.HasSuffix(name, ".tf.tpl")
}

func dirExists(path string) (bool, error) {
	st, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return st.IsDir(), nil
}

func copyFile(src, dst string) error {
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer out.Close()

	if _, err = io.Copy(out, in); err != nil {
		return err
	}
	if err := out.Close(); err != nil {
		return err
	}
	return nil
}

A pkg/hook/tfhook/tfhook_test.go => pkg/hook/tfhook/tfhook_test.go +68 -0
@@ 0,0 1,68 @@
package tfhook

import (
	"fmt"
	"os"
	"path/filepath"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/internal/testing"
	"github.com/stretchr/testify/require"
)

func TestHook(t *testing.T) {
	cases := []struct {
		root     string
		tfFile   []string // files that must be present
		noTfFile []string // files that must not be present
	}{
		{"valid", []string{"output.tf"}, nil},
		{"invalid", nil, []string{"noconfig.tf"}},
		{"perstage", []string{"file.tf"}, nil},
		{"template", []string{"file.tf"}, nil},
		{"none", nil, nil},
	}

	for i, c := range cases {
		t.Run(c.root, func(t *testing.T) {
			ess := testing.NewEssentials(t).
				RCDefaults(straw.RunConfig{
					RootDir:               filepath.Join("testdata", c.root),
					TerraformStageDir:     fmt.Sprintf("testdata/tmp/t%d", i),
					TerraformBootstrapDir: fmt.Sprintf("testdata/tmp/t%d-b", i),
				})

			defer func() {
				if t.Failed() {
					return
				}
				err := os.RemoveAll(ess.RC.TerraformStageDir)
				require.NoError(t, err)
				err = os.RemoveAll(ess.RC.TerraformBootstrapDir)
				require.NoError(t, err)
			}()

			h := Hook{
				RC: ess.RC,
				CR: ess.CR,
			}
			require.NoError(t, h.Run())

			if len(c.tfFile) > 0 {
				require.Equal(t, 1, len(ess.CR.Generators))
			} else {
				require.Equal(t, 0, len(ess.CR.Generators))
				return
			}

			require.NoError(t, os.MkdirAll(ess.RC.TerraformStageDir, 0755))
			err := h.Generate(ess.RC.TerraformStageDir, ess.Registries)
			require.NoError(t, err)

			for _, want := range c.tfFile {
				_, err := os.Stat(filepath.Join(ess.RC.TerraformStageDir, want))
				require.NoError(t, err)
			}
		})
	}
}

M pkg/integration_test.go => pkg/integration_test.go +85 -28
@@ 18,7 18,10 @@ import (
	"git.sr.ht/~mna/straw/pkg/errors"
	"git.sr.ht/~mna/straw/pkg/exec"
	"git.sr.ht/~mna/straw/pkg/hook/core"
	"git.sr.ht/~mna/straw/pkg/hook/tfhook"
	"git.sr.ht/~mna/straw/pkg/internal/testing"
	"git.sr.ht/~mna/straw/pkg/service/api"
	"git.sr.ht/~mna/straw/pkg/service/event"
	"git.sr.ht/~mna/straw/pkg/service/fn"
	"git.sr.ht/~mna/straw/pkg/service/queue"
	"git.sr.ht/~mna/straw/pkg/service/site"


@@ 44,7 47,6 @@ func TestIntegration(t *testing.T) {
	cleanUp := prepareAWSProvider(t)
	defer cleanUp()

	const stage = "test"
	os.Setenv("AWS_REGION", "us-east-1")

	dirs, err := ioutil.ReadDir("testdata")


@@ 63,6 65,21 @@ func TestIntegration(t *testing.T) {
			TF:       ess.TF,
			Progress: ioutil.Discard,
		})
		ess.Hook(&tfhook.Hook{
			RC: ess.RC,
			CR: ess.CR,
		})

		apiSvc := &api.Service{
			RC: ess.RC,
			CR: ess.CR,
		}
		ess.Service(apiSvc)
		ess.Service(apiSvc.PathMethod())
		ess.Service(&event.Service{
			RC: ess.RC,
			CR: ess.CR,
		})
		ess.Service(&fn.Service{
			RC: ess.RC,
			CR: ess.CR,


@@ 100,24 117,28 @@ func TestIntegration(t *testing.T) {
				if *leaveTempFiles || t.Failed() {
					return
				}
				os.RemoveAll(stageDir)
				os.RemoveAll(bootstrapDir)

				dirs, err := ioutil.ReadDir(tmpDir)
				require.NoError(t, err)
				for _, dir := range dirs {
					if !dir.IsDir() {
						continue
					}
					err = os.RemoveAll(filepath.Join(tmpDir, dir.Name()))
					require.NoError(t, err)
				}
			}()

			rc := straw.RunConfig{
				Stage:   stage,
				AppName: dir.Name(),

				RootDir:               rootDir,
				TerraformStageDir:     stageDir,
				TerraformBootstrapDir: bootstrapDir,

				TerraformVersionCondition:            "> 0.11.0",
				TerraformAWSProviderVersionCondition: "~> 1.43",
			}

			ess := testing.NewEssentials(t).
				RunConfig(rc).
				RCDefaults(rc).
				TFMock(nil,
					"bucket_terraform_state_region", "us-east-1",
					"bucket_terraform_state_name", "bucket",


@@ 126,31 147,67 @@ func TestIntegration(t *testing.T) {
			initEssentials(ess)
			installAWSProvider(t, stageDir)

			// assert a successful result, unless the directory of the
			// test ends with tilde ("~"), in which case assert an
			// error.
			assertFunc := assertSuccess(t, tfcmd, &buffer, stageDir, rootDir)
			if strings.HasSuffix(dir.Name(), "~") {
				assertFunc = assertError(t, tfcmd, &buffer, stageDir, rootDir)
			}
			ess.RunCmd(func(cmd *exec.Cmd, err error) {
				buffer.Reset()
				assertFunc(cmd, err)
			})
		})
	}
}

				require.IsType(t, &errors.ApplyError{}, err, "%s", err)
				ae := err.(*errors.ApplyError)
				require.Equal(t, straw.ErrApplyRequired, ae.Err)
func assertSuccess(t *testing.T, tfcmd *terraform.Cmd, buffer *bytes.Buffer, stageDir, rootDir string) func(*exec.Cmd, error) {
	return func(cmd *exec.Cmd, err error) {
		require.IsType(t, &errors.ApplyError{}, err, "%s", err)
		ae := err.(*errors.ApplyError)
		require.Equal(t, straw.ErrApplyRequired, ae.Err)

				require.NoError(t, tfcmd.Exec("fmt", stageDir), buffer.String())
				require.NoError(t, tfcmd.Exec("validate", stageDir), buffer.String())
		require.NoError(t, tfcmd.Exec("fmt", stageDir), buffer.String())
		require.NoError(t, tfcmd.Exec("validate", stageDir), buffer.String())

				// validate collected resources against golden file
				relativifyPaths(t, cmd.Registries.Resources)
				goldenFile := filepath.Join(rootDir, "golden.json")
				b, err := json.MarshalIndent(cmd.Registries.Resources, "", "  ")
				require.NoError(t, err)
				got := string(b)
				if *updateGoldenFiles {
					require.NoError(t, ioutil.WriteFile(goldenFile, b, 0644))
				}
				b, err = ioutil.ReadFile(goldenFile)
				require.NoError(t, err)
				want := string(b)
				require.JSONEq(t, want, got)
			})
		})
		// validate collected resources against golden file
		relativifyPaths(t, cmd.Registries.Resources)
		goldenFile := filepath.Join(rootDir, "golden.json")
		b, err := json.MarshalIndent(cmd.Registries.Resources, "", "  ")
		require.NoError(t, err)
		got := string(b)
		if *updateGoldenFiles {
			require.NoError(t, ioutil.WriteFile(goldenFile, b, 0644))
		}
		b, err = ioutil.ReadFile(goldenFile)
		require.NoError(t, err)
		want := string(b)
		require.JSONEq(t, want, got)
	}
}

func assertError(t *testing.T, tfcmd *terraform.Cmd, buffer *bytes.Buffer, stageDir, rootDir string) func(*exec.Cmd, error) {
	return func(cmd *exec.Cmd, err error) {
		require.Error(t, err)

		goldenFile := filepath.Join(rootDir, "golden.err.json")
		m := map[string]interface{}{
			"type":  fmt.Sprintf("%T", err),
			"error": err.Error(),
		}
		b, err := json.MarshalIndent(m, "", "  ")
		require.NoError(t, err)
		got := string(b)

		if *updateGoldenFiles {
			require.NoError(t, ioutil.WriteFile(goldenFile, b, 0644))
		}

		b, err = ioutil.ReadFile(goldenFile)
		require.NoError(t, err)
		want := string(b)
		require.JSONEq(t, want, got)
	}
}


M pkg/internal/testing/testing.go => pkg/internal/testing/testing.go +68 -0
@@ 1,11 1,13 @@
package testing

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	gotesting "testing"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/errors"
	"git.sr.ht/~mna/straw/pkg/exec"
)



@@ 190,6 192,31 @@ func NewEssentials(t *T) *Essentials {
	}
}

// CollRes assigns the CollectedResources on the Essentials, and
// builds the Resources and Environment registries accordingly.
func (e *Essentials) CollRes(cr *straw.CollectedResources) *Essentials {
	e.CR = cr
	e.Registries.Resources = make(map[string]straw.Resource)
	e.Registries.Environment = make(map[string]string)

	for _, r := range e.CR.Resources {
		typ, id := r.TypeID()
		typID := straw.FormatTypeID(typ, id)
		if _, ok := e.Registries.Resources[typID]; ok {
			e.T.Fatalf("%s: duplicate declaration", typID)
		}
		e.Registries.Resources[typID] = r

		for k, v := range r.Service().EnvVarMapping() {
			e.Registries.Environment[fmt.Sprintf("straw.%s.%s.%s", typ, id, k)] = r.TerraformAttr(v)
		}
	}
	e.Registries.Environment["straw.stage"] = e.RC.Stage
	e.Registries.Environment["straw.app"] = e.RC.AppName

	return e
}

// RCDefaults sets the RC to rc, with missing values adjusted
// to sane defaults for tests.
func (e *Essentials) RCDefaults(rc straw.RunConfig) *Essentials {


@@ 199,12 226,27 @@ func (e *Essentials) RCDefaults(rc straw.RunConfig) *Essentials {
	if rc.Stage == "" {
		rc.Stage = "test"
	}
	if rc.TerraformAWSProviderVersionCondition == "" {
		rc.TerraformAWSProviderVersionCondition = "~> 1.43"
	}
	if rc.TerraformVersionCondition == "" {
		rc.TerraformVersionCondition = "> 0.11"
	}
	if rc.TerraformStageDir == "" {
		rc.TerraformStageDir = filepath.Join("testdata", "tmp")
	}
	if rc.TerraformBootstrapDir == "" {
		rc.TerraformStageDir = filepath.Join("testdata", "tmp")
	}
	if rc.SrcDirName == "" {
		rc.SrcDirName = "src"
	}
	if rc.DistDirName == "" {
		rc.DistDirName = "dist"
	}
	if rc.ConfigFiles == nil {
		rc.ConfigFiles = straw.NewConfigFiles("test")
	}
	e.RC = rc
	return e
}


@@ 311,3 353,29 @@ func (e *Essentials) RunCmd(fn func(cmd *exec.Cmd, err error)) *Essentials {
	fn(cmd, err)
	return e
}

// RunValidators runs the validation on resources and generators
// that implement the Validator interface. It returns the first
// error encountered, if any.
func (e *Essentials) RunValidators() error {
	for _, r := range e.CR.Resources {
		if v, ok := r.(straw.Validator); ok {
			if err := v.Validate(e.Registries); err != nil {
				typ, id := r.TypeID()
				return &errors.ResourceError{
					Err:  err,
					Type: typ,
					ID:   id,
				}
			}
		}
	}
	for _, g := range e.CR.Generators {
		if v, ok := g.(straw.Validator); ok {
			if err := v.Validate(e.Registries); err != nil {
				return err
			}
		}
	}
	return nil
}

M pkg/service/api/config.go => pkg/service/api/config.go +36 -40
@@ 2,7 2,7 @@ package api

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/base64"
	"fmt"
	"os"
	"path/filepath"


@@ 18,7 18,6 @@ import (
)

type config struct {
	ID                 string `toml:"id"`
	Name               string `toml:"name"`
	StagePrefix        bool   `toml:"stage_prefix"`
	Description        string `toml:"description"`


@@ 28,8 27,6 @@ type config struct {
	Root struct {
		Methods []*pathMethod `toml:"methods"`
	} `toml:"root"`

	Tags map[string]string `toml:"tags"`
}

type pathPart struct {


@@ 65,22 62,19 @@ func (s *Service) applyAPIConfig(sc service.ScanConfig, res straw.Resource, file
	}

	api := res.(*Resource)
	if c.ID != "" {
		api.ID = c.ID
	}
	if c.Name != "" {
	if meta.IsDefined("name") {
		api.Name = c.Name
	}
	if meta.IsDefined("stage_prefix") {
		api.StagePrefix = c.StagePrefix
	}
	if c.Description != "" {
	if meta.IsDefined("description") {
		api.Description = c.Description
	}
	if c.EndpointType != "" {
	if meta.IsDefined("endpoint_type") {
		api.EndpointType = c.EndpointType
	}
	if c.MinCompressionSize != "" {
	if meta.IsDefined("minimum_compression_size") {
		// first parse as a normal int, as -1 is accepted.
		sz, err := strconv.ParseInt(c.MinCompressionSize, 10, 0)
		if err != nil {


@@ 92,14 86,11 @@ func (s *Service) applyAPIConfig(sc service.ScanConfig, res straw.Resource, file
		api.MinCompressionSize = sz
	}
	if meta.IsDefined("root") {
		part := PathPart{ID: pathPartID("/"), Part: "/"}
		if err := applyMethodsConfig(&part, c.Root.Methods); err != nil {
		part := &PathPart{api: api, ID: pathPartID(api.ID, ""), Part: "/"}
		if err := s.applyMethodsConfig(part, c.Root.Methods); err != nil {
			return err
		}
		api.Paths = []PathPart{part}
	}
	if meta.IsDefined("tags") {
		api.Tags = c.Tags
		api.Paths = []*PathPart{part}
	}

	return nil


@@ 123,37 114,33 @@ func (s *Service) postAPI(sc service.ScanConfig, res straw.Resource) error {
		relPath := strings.TrimPrefix(path, sc.Src)
		parentPath := filepath.ToSlash(filepath.Dir(relPath))
		curPath := filepath.ToSlash(relPath)
		pp := PathPart{
			ID:       pathPartID(curPath),
			ParentID: pathPartID(parentPath),
		pp := &PathPart{
			api:      api,
			ID:       pathPartID(api.ID, curPath),
			ParentID: pathPartID(api.ID, parentPath),
			Part:     fi.Name(),
		}

		// read and apply config files
		files := []string{
			filepath.Join(path, straw.ResourceConfigFilename),
			filepath.Join(path, fmt.Sprintf(straw.FmtStageResourceConfigFilename, sc.Stage)),
		}
		configured := false
		for _, file := range files {
		count, err := s.RC.ConfigFiles.EachIn(path, func(file string) error {
			var c pathPart
			_, err := toml.DecodeFile(file, &c)
			meta, err := toml.DecodeFile(file, &c)
			if err != nil {
				if os.IsNotExist(err) {
					continue
				}
				return err
			}
			if c.Part != "" {
			if meta.IsDefined("part") {
				pp.Part = c.Part
			}
			if err := applyMethodsConfig(&pp, c.Methods); err != nil {
				return err
			if meta.IsDefined("methods") {
				pp.Methods = nil
				return s.applyMethodsConfig(pp, c.Methods)
			}
			configured = true
			return nil
		})
		if err != nil {
			return err
		}

		if configured {
		if count > 0 {
			api.Paths = append(api.Paths, pp)
		}



@@ 163,10 150,13 @@ func (s *Service) postAPI(sc service.ScanConfig, res straw.Resource) error {
	return err
}

func applyMethodsConfig(pp *PathPart, methods []*pathMethod) error {
func (s *Service) applyMethodsConfig(pp *PathPart, methods []*pathMethod) error {
	for _, m := range methods {
		pm := &PathMethod{
			Method: m.Method,
			ID:       fmt.Sprintf("%s_%s", m.Method, pp.ID),
			Method:   m.Method,
			service:  s.PathMethod(),
			pathPart: pp,
		}
		if m.AuthType != "" {
			pm.Auth = &Auth{Type: m.AuthType, ID: m.AuthID}


@@ 187,10 177,16 @@ func applyMethodsConfig(pp *PathPart, methods []*pathMethod) error {
	return nil
}

func pathPartID(path string) string {
func pathPartID(apiID, path string) string {
	if path == "/" {
		// special-case, this is the root resource's ID, return
		// empty string.
		return ""
	}
	h := sha256.New()
	if _, err := h.Write([]byte(path)); err != nil {
		return ""
	}
	return hex.EncodeToString(h.Sum(nil))
	b64 := base64.RawURLEncoding.EncodeToString(h.Sum(nil))
	return fmt.Sprintf("%s_%s", apiID, b64)
}

M pkg/service/api/resource.go => pkg/service/api/resource.go +176 -11
@@ 3,13 3,17 @@ package api
import (
	"errors"
	"fmt"
	"path"
	"strings"
	"time"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/service"
)

var (
	_ straw.Resource  = (*Resource)(nil)
	_ straw.Resource  = (*PathMethod)(nil)
	_ straw.Validator = (*Resource)(nil)
)



@@ 21,9 25,8 @@ type Resource struct {
	Description        string
	EndpointType       string
	MinCompressionSize int64

	Paths []PathPart
	Tags  map[string]string
	Policy             *straw.Policy
	Paths              []*PathPart

	service *Service
}


@@ 34,13 37,30 @@ type PathPart struct {
	ParentID string
	Part     string
	Methods  []*PathMethod

	// arnPath is filled during the Validation step.
	arnPath string
	api     *Resource
}

// PathMethod is a single method for a path part.
// PathMethod is a single method for a path part. It is a
// straw.Resource itself, with a PathMethodService as parent
// service.
type PathMethod struct {
	ID      string
	Method  string
	Auth    *Auth
	Handler *Handler

	service  *PathMethodService
	pathPart *PathPart
}

func (m *PathMethod) arnMethod() string {
	if m.Method == "ANY" {
		return "*"
	}
	return m.Method
}

// Auth identifies an authentication method.


@@ 78,12 98,12 @@ func (r *Resource) Service() straw.Service {

// GrantAccessFrom grants access to r from other.
func (r *Resource) GrantAccessFrom(other straw.Resource) error {
	return errors.New("not implemented")
	return errors.New("access must be granted to an API path method")
}

// RequestAccessTo requests access to other from r.
func (r *Resource) RequestAccessTo(other straw.Resource) error {
	return errors.New("not implemented")
	return errors.New("access must be requested for an API path method")
}

// PolicyStatement returns the policy statement to allow access to


@@ 91,10 111,9 @@ func (r *Resource) RequestAccessTo(other straw.Resource) error {
func (r *Resource) PolicyStatement() *straw.PolicyStatement {
	typID := straw.FormatTypeID(r.TypeID())
	return &straw.PolicyStatement{
		Sid:    fmt.Sprintf("allow_%s", typID),
		Effect: straw.Allow,
		// TODO: should be /restapis/api-id/*, not the name
		Resources: []string{fmt.Sprintf("arn:aws:apigateway:*::/restapis/%s/*", r.EffectiveName())},
		Sid:       fmt.Sprintf("allow_%s", typID),
		Effect:    straw.Allow,
		Resources: []string{fmt.Sprintf("arn:aws:apigateway:${var.region}::/restapis/%s/*", r.TerraformAttr("id"))},
		Actions:   []string{"apigateway:*"},
	}
}


@@ 105,8 124,154 @@ func (r *Resource) TerraformAttr(attr string) string {
	return fmt.Sprintf("${aws_api_gateway_rest_api.%s.%s}", r.ID, attr)
}

// TypeID returns the type and ID of that resource, which
// as a tuple uniquely identifies it.
func (m *PathMethod) TypeID() (string, string) {
	return m.service.Type(), m.ID
}

// Service returns the parent Service of this Resource.
func (m *PathMethod) Service() straw.Service {
	return m.service
}

// GrantAccessFrom grants access to r from other.
func (m *PathMethod) GrantAccessFrom(other straw.Resource) error {
	if other.Service().Type() == "func" {
		return other.RequestAccessTo(m)
	}
	return fmt.Errorf("cannot grant access to PathMethod from %s", other.Service().Type())
}

// RequestAccessTo requests access to other from m.
func (m *PathMethod) RequestAccessTo(other straw.Resource) error {
	return other.GrantAccessFrom(m)
}

// PolicyStatement returns the policy statement to allow access to
// this api's path method.
func (m *PathMethod) PolicyStatement() *straw.PolicyStatement {
	panic("PolicyStatement unsupported on the PathMethod")
}

// TerraformAttr returns a terraform reference for this resource's
// attribute.
func (m *PathMethod) TerraformAttr(attr string) string {
	if attr != "arn" {
		panic("PathMethod resource: unsupported terraform attribute: " + attr)
	}
	// Format after the execution arn is /METHOD/PATH, and method
	// is "*" for ANY, and path is "*" for any part inside "{...}".
	pp := m.pathPart
	api := pp.api
	// NOTE: using aws_api_gateway_deployment.execution_arn does NOT work,
	// seemingly because of the stage part (which must be *).
	return fmt.Sprintf("${aws_api_gateway_rest_api.%s.execution_arn}/*/%s%s", api.ID, m.arnMethod(), pp.arnPath)
}

// Validate validates the resource and returns an error if
// it is invalid.
func (r *Resource) Validate(reg *straw.Registries) error {
	return errors.New("not implemented")
	if err := service.ValidateTerraformID(r.ID); err != nil {
		return err
	}
	if r.Policy != nil {
		if err := service.ValidatePolicy(r.Policy); err != nil {
			return err
		}
	}

	// build a lookup map of ID to PathPart to resolve full paths
	pathPartLookup := make(map[string]*PathPart, len(r.Paths))
	for _, pp := range r.Paths {
		pathPartLookup[pp.ID] = pp
	}
	for _, pp := range r.Paths {
		if err := pp.validate(reg, pathPartLookup); err != nil {
			return err
		}
	}
	for _, pp := range r.Paths {
		pp.arnPath = resolveArnPath(pp, pathPartLookup)
	}
	return nil
}

func (pp *PathPart) validate(reg *straw.Registries, pathPartLookup map[string]*PathPart) error {
	if err := service.ValidateTerraformID(pp.ID); err != nil {
		return err
	}
	if pp.ParentID != "" {
		if _, ok := pathPartLookup[pp.ParentID]; !ok {
			return fmt.Errorf("unknown parent path id: %s", pp.ParentID)
		}
	}
	for _, m := range pp.Methods {
		if err := m.validate(reg); err != nil {
			return err
		}
	}
	return nil
}

func (m *PathMethod) validate(reg *straw.Registries) error {
	if m.Auth != nil {
		if err := m.Auth.validate(reg); err != nil {
			return err
		}
	}

	if m.Handler != nil {
		if err := m.Handler.validate(reg); err != nil {
			return err
		}
		typID := straw.FormatTypeID(m.Handler.Type, m.Handler.ID)
		rr := reg.Resources[typID]
		if err := m.RequestAccessTo(rr); err != nil {
			return err
		}
	}
	return nil
}

func (a *Auth) validate(reg *straw.Registries) error {
	if reg.Services[a.Type] == nil {
		return fmt.Errorf("invalid auth type: %s", a.Type)
	}
	// TODO: validate type is one of the valid auth types
	typID := straw.FormatTypeID(a.Type, a.ID)
	if reg.Resources[typID] == nil {
		return fmt.Errorf("unknown %s id: %s", a.Type, a.ID)
	}
	return nil
}

func (h *Handler) validate(reg *straw.Registries) error {
	if reg.Services[h.Type] == nil {
		return fmt.Errorf("invalid handler type: %s", h.Type)
	}
	// only "func" is supported for now
	if h.Type != "func" {
		return fmt.Errorf("unsupported handler type: %s", h.Type)
	}
	typID := straw.FormatTypeID(h.Type, h.ID)
	if reg.Resources[typID] == nil {
		return fmt.Errorf("unknown %s id: %s", h.Type, h.ID)
	}
	return nil
}

func resolveArnPath(pp *PathPart, pathPartLookup map[string]*PathPart) string {
	part := pp.Part
	if strings.HasPrefix(part, "{") && strings.HasSuffix(part, "}") {
		part = "*"
	}
	if pp.Part == "/" {
		return part
	}
	if pp.ParentID == "" {
		return "/" + part
	}
	parent := pathPartLookup[pp.ParentID]
	return path.Join(resolveArnPath(parent, pathPartLookup), part)
}

M pkg/service/api/service.go => pkg/service/api/service.go +44 -3
@@ 8,6 8,7 @@ import (

var (
	_ straw.Service   = (*Service)(nil)
	_ straw.Service   = (*PathMethodService)(nil)
	_ straw.Generator = (*Service)(nil)
)



@@ 15,6 16,35 @@ var (
type Service struct {
	RC straw.RunConfig
	CR *straw.CollectedResources

	pms *PathMethodService
}

// PathMethodService is the service implementation for api
// path method parts.
// It is identical to Service except for the Type, which returns
// "apipathmethod" instead of "api".
type PathMethodService struct {
	*Service
}

// Type returns the type name for this service.
func (s *PathMethodService) Type() string {
	return "apipathmethod"
}

// Scan is a no-op for PathMethodService, the scanning of all
// paths and methods is done via the Service.Scan method.
func (s *PathMethodService) Scan(src, dist string) error {
	return nil
}

// PathMethod returns the straw.Service for PathMethods.
func (s *Service) PathMethod() *PathMethodService {
	if s.pms == nil {
		s.pms = &PathMethodService{s}
	}
	return s.pms
}

// EnvVarMapping returns the mapping of straw attributes to


@@ 36,9 66,10 @@ func (s *Service) Principal() *straw.PolicyPrincipal {
// provided src directory.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		NewFunc:   s.newAPI,
		ApplyFunc: s.applyAPIConfig,
		PostFunc:  s.postAPI,
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newAPI,
		ApplyFunc:   s.applyAPIConfig,
		PostFunc:    s.postAPI,
	}
	sc := service.ScanConfig{
		Src:   src,


@@ 48,6 79,16 @@ func (s *Service) Scan(src, dist string) error {
	res, err := rs.Scan(sc)
	if len(res) > 0 {
		s.CR.Resources = append(s.CR.Resources, res...)
		for _, r := range res {
			r, ok := r.(*Resource)
			if ok {
				for _, pp := range r.Paths {
					for _, m := range pp.Methods {
						s.CR.Resources = append(s.CR.Resources, m)
					}
				}
			}
		}
		s.CR.Generators = append(s.CR.Generators, s)
	}
	return err

M pkg/service/api/service_test.go => pkg/service/api/service_test.go +195 -24
@@ 3,10 3,10 @@ package api
import (
	"path/filepath"
	"sort"
	"testing"
	"time"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/internal/testing"
	"github.com/stretchr/testify/require"
)



@@ 15,7 15,10 @@ func TestService_Scan(t *testing.T) {
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test"},
		RC: straw.RunConfig{
			Stage:       "test",
			ConfigFiles: straw.NewConfigFiles("test"),
		},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)


@@ 26,18 29,19 @@ func TestService_Scan(t *testing.T) {
			ID:          "api",
			Name:        "api",
			StagePrefix: true,
			Paths: []PathPart{
			Paths: []*PathPart{
				{
					ID:       pathPartID("/base"),
					ParentID: pathPartID("/"),
					ID:       pathPartID("api", "/base"),
					ParentID: "",
					Part:     "base",
				},
				{
					ID:       pathPartID("/base/leaf"),
					ParentID: pathPartID("/base"),
					ID:       pathPartID("api", "/base/leaf"),
					ParentID: pathPartID("api", "/base"),
					Part:     "leaf",
					Methods: []*PathMethod{
						{
							ID:     "GET_" + pathPartID("api", "/base/leaf"),
							Method: "GET",
							Handler: &Handler{
								Type:    "func",


@@ 46,6 50,7 @@ func TestService_Scan(t *testing.T) {
							},
						},
						{
							ID:     "POST_" + pathPartID("api", "/base/leaf"),
							Method: "POST",
							Handler: &Handler{
								Type:    "func",


@@ 56,30 61,31 @@ func TestService_Scan(t *testing.T) {
					},
				},
				{
					ID:       pathPartID("/otherbase"),
					ParentID: pathPartID("/"),
					ID:       pathPartID("api", "/otherbase"),
					ParentID: "",
					Part:     "otherbase",
				},
				{
					ID:       pathPartID("/otherbase/leaf"),
					ParentID: pathPartID("/otherbase"),
					ID:       pathPartID("api", "/otherbase/leaf"),
					ParentID: pathPartID("api", "/otherbase"),
					Part:     "leaf",
				},
			},
		},
		{
			ID:                 "overrideapi",
			ID:                 "apiconfig",
			Name:               "theapi",
			StagePrefix:        true,
			Description:        "a description",
			EndpointType:       "REGIONAL",
			MinCompressionSize: 10240,
			Paths: []PathPart{
			Paths: []*PathPart{
				{
					ID:   pathPartID("/"),
					ID:   pathPartID("apiconfig", ""),
					Part: "/",
					Methods: []*PathMethod{
						{
							ID:     "GET_" + pathPartID("apiconfig", ""),
							Method: "GET",
							Handler: &Handler{
								Type:    "func",


@@ 88,6 94,7 @@ func TestService_Scan(t *testing.T) {
							},
						},
						{
							ID:     "DELETE_" + pathPartID("apiconfig", ""),
							Method: "DELETE",
							Handler: &Handler{
								Type: "func",


@@ 103,18 110,19 @@ func TestService_Scan(t *testing.T) {
			Name:         "testonlyapi",
			StagePrefix:  true,
			EndpointType: "REGIONAL",
			Paths: []PathPart{
			Paths: []*PathPart{
				{
					ID:       pathPartID("/base"),
					ParentID: pathPartID("/"),
					ID:       pathPartID("testonlyapi", "/base"),
					ParentID: "",
					Part:     "base",
				},
				{
					ID:       pathPartID("/base/{proxy+}"),
					ParentID: pathPartID("/base"),
					ID:       pathPartID("testonlyapi", "/base/{proxy+}"),
					ParentID: pathPartID("testonlyapi", "/base"),
					Part:     "{proxy+}",
					Methods: []*PathMethod{
						{
							ID:     "ANY_" + pathPartID("testonlyapi", "/base/{proxy+}"),
							Method: "ANY",
							Handler: &Handler{
								Type: "func",


@@ 127,16 135,167 @@ func TestService_Scan(t *testing.T) {
		},
	}

	got := make([]*Resource, len(s.CR.Resources))
	for i, r := range s.CR.Resources {
		got[i] = r.(*Resource)
		// nil out the service, unnecessary for the test
		got[i].service = nil
	got := make([]*Resource, 0, len(s.CR.Resources))
	for _, r := range s.CR.Resources {
		switch r := r.(type) {
		case *Resource:
			got = append(got, r)
		}
	}
	orderApisLists(expect, got)
	require.Equal(t, expect, got)
}

func TestResolveArnPathWithRoot(t *testing.T) {
	paths := []*PathPart{
		{
			ID:       pathPartID("a", ""),
			ParentID: "",
			Part:     "/",
		},
		{
			ID:       pathPartID("a", "/base"),
			ParentID: "",
			Part:     "base",
		},
		{
			ID:       pathPartID("a", "/base/leaf"),
			ParentID: pathPartID("a", "/base"),
			Part:     "leaf",
		},
		{
			ID:       pathPartID("a", "/other"),
			ParentID: "",
			Part:     "other",
		},
		{
			ID:       pathPartID("a", "/other/{param}"),
			ParentID: pathPartID("a", "/other"),
			Part:     "{param}",
		},
		{
			ID:       pathPartID("a", "/other/{param}/sub"),
			ParentID: pathPartID("a", "/other/{param}"),
			Part:     "sub",
		},
		{
			ID:       pathPartID("a", "/third"),
			ParentID: "",
			Part:     "third",
		},
		{
			ID:       pathPartID("a", "/third/{proxy+}"),
			ParentID: pathPartID("a", "/third"),
			Part:     "{proxy+}",
		},
	}
	want := []string{
		"/",
		"/base",
		"/base/leaf",
		"/other",
		"/other/*",
		"/other/*/sub",
		"/third",
		"/third/*",
	}

	pathPartLookup := make(map[string]*PathPart, len(paths))
	for _, p := range paths {
		pathPartLookup[p.ID] = p
	}
	var got []string
	for _, p := range paths {
		got = append(got, resolveArnPath(p, pathPartLookup))
	}
	require.Equal(t, want, got)
}

func TestResolveArnPathWithoutRoot(t *testing.T) {
	paths := []*PathPart{
		{
			ID:       pathPartID("a", "/base"),
			ParentID: "",
			Part:     "base",
		},
		{
			ID:       pathPartID("a", "/base/leaf"),
			ParentID: pathPartID("a", "/base"),
			Part:     "leaf",
		},
		{
			ID:       pathPartID("a", "/other"),
			ParentID: "",
			Part:     "other",
		},
		{
			ID:       pathPartID("a", "/other/{param}"),
			ParentID: pathPartID("a", "/other"),
			Part:     "{param}",
		},
		{
			ID:       pathPartID("a", "/other/{param}/sub"),
			ParentID: pathPartID("a", "/other/{param}"),
			Part:     "sub",
		},
		{
			ID:       pathPartID("a", "/third"),
			ParentID: "",
			Part:     "third",
		},
		{
			ID:       pathPartID("a", "/third/{proxy+}"),
			ParentID: pathPartID("a", "/third"),
			Part:     "{proxy+}",
		},
	}
	want := []string{
		"/base",
		"/base/leaf",
		"/other",
		"/other/*",
		"/other/*/sub",
		"/third",
		"/third/*",
	}

	pathPartLookup := make(map[string]*PathPart, len(paths))
	for _, p := range paths {
		pathPartLookup[p.ID] = p
	}
	var got []string
	for _, p := range paths {
		got = append(got, resolveArnPath(p, pathPartLookup))
	}
	require.Equal(t, want, got)
}

func TestResolveArnPathProxyRoot(t *testing.T) {
	paths := []*PathPart{
		{
			ID:       pathPartID("a", ""),
			ParentID: "",
			Part:     "/",
		},
		{
			ID:       pathPartID("a", "/{proxy+}"),
			ParentID: "",
			Part:     "{proxy+}",
		},
	}
	want := []string{"/", "/*"}

	pathPartLookup := make(map[string]*PathPart, len(paths))
	for _, p := range paths {
		pathPartLookup[p.ID] = p
	}
	var got []string
	for _, p := range paths {
		got = append(got, resolveArnPath(p, pathPartLookup))
	}
	require.Equal(t, want, got)
}

func orderApisLists(apis ...[]*Resource) {
	// sort lists to make result deterministic
	for _, v := range apis {


@@ 146,10 305,22 @@ func orderApisLists(apis ...[]*Resource) {
		})

		for _, vv := range v {
			vv.service = nil
			sort.Slice(vv.Paths, func(i, j int) bool {
				l, r := vv.Paths[i], vv.Paths[j]
				return l.ID < r.ID
			})
			for _, pp := range vv.Paths {
				pp.api = nil
				sort.Slice(pp.Methods, func(i, j int) bool {
					l, r := pp.Methods[i], pp.Methods[j]
					return l.ID < r.ID
				})
				for _, m := range pp.Methods {
					m.pathPart = nil
					m.service = nil
				}
			}
		}
	}
}

M pkg/service/api/testdata/src/apiconfig/straw.toml => pkg/service/api/testdata/src/apiconfig/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "overrideapi"
name = "theapi"
description = "a description"
endpoint_type = "REGIONAL"

A pkg/service/auth/config.go => pkg/service/auth/config.go +101 -0
@@ 0,0 1,101 @@
package auth

import (
	"path/filepath"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/service"

	"github.com/BurntSushi/toml"
)

type config struct {
	Name              string            `toml:"name"`
	MFA               string            `toml:"mfa"`
	AliasAttrs        []string          `toml:"alias_attributes"`
	UsernameAttrs     []string          `toml:"username_attributes"`
	AutoVerifiedAttrs []string          `toml:"auto_verified_attributes"`
	ClientName        string            `toml:"client_name"`
	Tags              map[string]string `toml:"tags"`

	VerificationMessage struct {
		DefaultEmailOption string `toml:"default_email_option"`
		EmailMessage       string `toml:"email_message"`
		EmailSubject       string `toml:"email_subject"`
	} `toml:"verification_message"`

	Admin struct {
		AllowAdminCreateUserOnly bool   `toml:"allow_admin_create_user_only"`
		UnusedAccountValidity    string `toml:"unused_account_validity"`
		InviteEmailMessage       string `toml:"invite_email_message"`
		InviteEmailSubject       string `toml:"invite_email_subject"`
	} `toml:"admin"`

	Device struct {
		ChallengeRequiredOnNewDevice     bool `toml:"challenge_required_on_new_device"`
		DeviceOnlyRememberedOnUserPrompt bool `toml:"device_only_remembered_on_user_prompt"`
	} `toml:"device"`

	PasswordPolicy struct {
		MinLength        int  `toml:"minimum_length"`
		RequireLowercase bool `toml:"require_lowercase"`
		RequireNumbers   bool `toml:"require_numbers"`
		RequireSymbols   bool `toml:"require_symbols"`
		RequireUppercase bool `toml:"require_uppercase"`
	} `toml:"password_policy"`

	Triggers struct {
		CreateAuthChallenge         string `toml:"create_auth_challenge"`
		CustomMessage               string `toml:"custom_message"`
		DefineAuthChallenge         string `toml:"define_auth_challenge"`
		PostAuthentication          string `toml:"post_authentication"`
		PostConfirmation            string `toml:"post_confirmation"`
		PreAuthentication           string `toml:"pre_authentication"`
		PreSignUp                   string `toml:"pre_sign_up"`
		PreTokenGeneration          string `toml:"pre_token_generation"`
		UserMigration               string `toml:"user_migration"`
		VerifyAuthChallengeResponse string `toml:"verify_auth_challenge_response"`
	} `toml:"triggers"`

	SchemaAttrs []struct {
		DataType      string `toml:"data_type"`
		DeveloperOnly bool   `toml:"developer_only"`
		Mutable       bool   `toml:"mutable"`
		Name          string `toml:"string"`
		Required      bool   `toml:"required"`
		MinLength     int    `toml:"min_length"`
		MaxLength     int    `toml:"max_length"`
		MinValue      int    `toml:"min_value"`
		MaxValue      int    `toml:"max_value"`
	} `toml:"schema_attributes"`
}

func (s *Service) newAuth(sc service.ScanConfig) straw.Resource {
	baseDir := filepath.Base(sc.Src)
	return &Resource{
		ID:          baseDir,
		Name:        baseDir,
		StagePrefix: true,
		ClientName:  s.RC.AppName,

		service: s,
	}
}

func (s *Service) applyAuthConfig(sc service.ScanConfig, res straw.Resource, file string) error {
	var c config
	meta, err := toml.DecodeFile(file, &c)
	if err != nil {
		return err
	}

	a := res.(*Resource)
	if meta.IsDefined("name") {
		a.Name = c.Name
	}
	// TODO: complete configuration...
	if meta.IsDefined("tags") {
		a.Tags = c.Tags
	}
	return nil
}

A pkg/service/auth/resource.go => pkg/service/auth/resource.go +28 -0
@@ 0,0 1,28 @@
package auth

import (
	"git.sr.ht/~mna/straw"
)

var (
	_ straw.Resource  = (*Resource)(nil)
	_ straw.Validator = (*Resource)(nil)
)

// Resource represents an auth resource.
type Resource struct {
	ID          string
	Name        string
	StagePrefix bool
	MFA         bool
	ClientName  string

	AliasAttrs        []string
	UsernameAttrs     []string
	AutoVerifiedAttrs []string
	Tags              map[string]string

	// TODO: complete...

	service *Service
}

A pkg/service/auth/service.go => pkg/service/auth/service.go +76 -0
@@ 0,0 1,76 @@
package auth

import (
	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/service"
	"git.sr.ht/~mna/straw/pkg/template"
)

var (
	_ straw.Service   = (*Service)(nil)
	_ straw.Generator = (*Service)(nil)
)

var envVarMapping = map[string]string{
	"name": "name",
	"arn":  "arn",
	"id":   "id",
}

// Service is the service implementation for auth.
type Service struct {
	RC straw.RunConfig
	CR *straw.CollectedResources
}

// EnvVarMapping returns the mapping of straw attributes to
// terraform attributes.
func (s *Service) EnvVarMapping() map[string]string {
	return envVarMapping
}

// Principal returns the policy principal for the auth
// service.
func (s *Service) Principal() *straw.PolicyPrincipal {
	return &straw.PolicyPrincipal{
		Type:        "Service",
		Identifiers: []string{"cognito-idp.amazonaws.com"},
	}
}

// Scan generates resources for this service by reading the
// provided src directory.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newAuth,
		ApplyFunc:   s.applyAuthConfig,
	}
	sc := service.ScanConfig{
		Src:   src,
		Dist:  dist,
		Stage: s.RC.Stage,
	}
	res, err := rs.Scan(sc)
	if len(res) > 0 {
		s.CR.Resources = append(s.CR.Resources, res...)
		s.CR.Generators = append(s.CR.Generators, s)
	}
	return err
}

// Generate implements a straw.Generator for the auth.
// It generates the auth config.
func (s *Service) Generate(dir string, reg *straw.Registries) error {
	t := template.New("auth.tf.tpl").
		FilteredResources("Auths", s.CR.Resources, s.Type()).
		ResourcesMap(reg.Resources).
		EnvMap(reg.Environment)

	return t.Execute(dir)
}

// Type returns the type name of this service.
func (s *Service) Type() string {
	return "auth"
}

A pkg/service/event/config.go => pkg/service/event/config.go +136 -0
@@ 0,0 1,136 @@
package event

import (
	"encoding/json"
	"errors"
	"fmt"
	"path/filepath"
	"regexp"
	"strconv"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/service"

	"github.com/BurntSushi/toml"
)

type config struct {
	Name        string `toml:"name"`
	StagePrefix bool   `toml:"stage_prefix"`
	IsPrefix    bool   `toml:"is_prefix"`
	Description string `toml:"description"`
	Role        string `toml:"role"`
	Enabled     bool   `toml:"enabled"`

	Schedule struct {
		Type  string `toml:"type"`
		Value string `toml:"value"`
	} `toml:"schedule"`

	RawEventPattern string `toml:"raw_event_pattern"`

	Targets []struct {
		TargetID string `toml:"target_id"`
		Type     string `toml:"type"`
		ID       string `toml:"id"`
	} `toml:"targets"`
}

func (s *Service) newEvent(sc service.ScanConfig) straw.Resource {
	baseDir := filepath.Base(sc.Src)
	return &Resource{
		ID:          baseDir,
		Name:        baseDir,
		StagePrefix: true,
		Enabled:     true,

		service: s,
	}
}

func (s *Service) applyEventConfig(sc service.ScanConfig, res straw.Resource, file string) error {
	var c config
	meta, err := toml.DecodeFile(file, &c)
	if err != nil {
		return err
	}

	ev := res.(*Resource)
	if meta.IsDefined("name") {
		ev.Name = c.Name
	}
	if meta.IsDefined("stage_prefix") {
		ev.StagePrefix = c.StagePrefix
	}
	if meta.IsDefined("is_prefix") {
		ev.IsPrefix = c.IsPrefix
	}
	if meta.IsDefined("description") {
		ev.Description = c.Description
	}
	if meta.IsDefined("role") {
		ev.RoleName = c.Role
	}
	if meta.IsDefined("enabled") {
		ev.Enabled = c.Enabled
	}
	if meta.IsDefined("raw_event_pattern") {
		if c.RawEventPattern == "" {
			ev.EventPattern = nil
		} else {
			if !json.Valid([]byte(c.RawEventPattern)) {
				return errors.New("invalid JSON in raw_event_pattern configuration")
			}
			ev.EventPattern = json.RawMessage(c.RawEventPattern)
		}
	}

	if meta.IsDefined("schedule") {
		expr := parseRateExpr(c.Schedule.Value)
		ev.Schedule = &Schedule{
			Type:  c.Schedule.Type,
			Value: expr,
		}
	}

	if meta.IsDefined("targets") {
		ev.Targets = make([]*Target, len(c.Targets))
		for i, t := range c.Targets {
			ev.Targets[i] = &Target{
				TargetID: t.TargetID,
				Type:     t.Type,
				ID:       t.ID,
			}
		}
	}

	return nil
}

var (
	rxShortRate     = regexp.MustCompile(`^\s*(\d+)([mhd])\s*$`)
	shortToLongUnit = map[string][]string{
		"m": {"minute", "minutes"},
		"h": {"hour", "hours"},
		"d": {"day", "days"},
	}
)

// parses a rate-type schedule expression into a format supported
// by AWS. Specifically, adds support for short "m", "h" and "d"
// suffixes for "minute(s)", "hour(s)" and "day(s)".
func parseRateExpr(expr string) string {
	m := rxShortRate.FindStringSubmatch(expr)
	if m == nil {
		// return as-is, assumed to be already in AWS format
		return expr
	}
	n, _ := strconv.Atoi(m[1])
	unit := m[2]

	ix := 0
	if n > 1 {
		ix = 1
	}
	return fmt.Sprintf("%d %s", n, shortToLongUnit[unit][ix])
}

A pkg/service/event/config_test.go => pkg/service/event/config_test.go +32 -0
@@ 0,0 1,32 @@
package event

import (
	"testing"

	"github.com/stretchr/testify/require"
)

func TestParseRateExpr(t *testing.T) {
	cases := []struct {
		in, out string
	}{
		{"", ""},
		{"5", "5"},
		{"5 minutes", "5 minutes"},
		{"1 hour", "1 hour"},
		{"1h", "1 hour"},
		{"2h", "2 hours"},
		{" 1d ", "1 day"},
		{" 100d ", "100 days"},
		{"1a", "1a"},
		{" 1h ", "1 hour"},
		{" 100h ", "100 hours"},
		{" 1 h ", " 1 h "},
	}
	for _, c := range cases {
		t.Run(c.in, func(t *testing.T) {
			got := parseRateExpr(c.in)
			require.Equal(t, c.out, got)
		})
	}
}

A pkg/service/event/resource.go => pkg/service/event/resource.go +141 -0
@@ 0,0 1,141 @@
package event

import (
	"errors"
	"fmt"
	"regexp"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/service"
)

var (
	_ straw.Resource  = (*Resource)(nil)
	_ straw.Validator = (*Resource)(nil)
)

// Resource represents an event resource.
type Resource struct {
	ID           string
	Name         string
	StagePrefix  bool
	IsPrefix     bool
	Description  string
	RoleName     string
	Enabled      bool
	EventPattern interface{}

	Schedule *Schedule
	Targets  []*Target

	service *Service
}

// Schedule defines a scheduled event.
type Schedule struct {
	Type  string
	Value string
}

// Target defines a target for the event.
type Target struct {
	TargetID string
	Type     string
	ID       string
}

// EffectiveName returns the name to use in this specific
// run, properly prefixed with the stage name if requested.
func (r *Resource) EffectiveName() string {
	if r.StagePrefix {
		return r.service.RC.Stage + "-" + r.Name
	}
	return r.Name
}

// TypeID returns the type and ID of that resource, which
// as a tuple uniquely identifies it.
func (r *Resource) TypeID() (string, string) {
	return r.service.Type(), r.ID
}

// Service returns the parent Service of this Resource.
func (r *Resource) Service() straw.Service {
	return r.service
}

// GrantAccessFrom grants access to r from other.
func (r *Resource) GrantAccessFrom(other straw.Resource) error {
	return errors.New("cannot grant access to an event")
}

// RequestAccessTo requests access to other from r.
func (r *Resource) RequestAccessTo(other straw.Resource) error {
	return other.GrantAccessFrom(r)
}

// PolicyStatement returns the policy statement to allow access to
// this event.
func (r *Resource) PolicyStatement() *straw.PolicyStatement {
	// TODO: probably doesn't make sense, cannot grant access to
	// an event.
	panic("cannot generate a policy to access an event")
}

// TerraformAttr returns a terraform reference for this resource's
// attribute.
func (r *Resource) TerraformAttr(attr string) string {
	return fmt.Sprintf("${aws_cloudwatch_event_rule.%s.%s}", r.ID, attr)
}

// Validate validates the resource and returns an error if
// it is invalid.
func (r *Resource) Validate(reg *straw.Registries) error {
	if err := service.ValidateTerraformID(r.ID); err != nil {
		return err
	}
	if err := validateName(r.Name); err != nil {
		return err
	}
	if r.Schedule != nil {
		if r.Schedule.Type != "cron" && r.Schedule.Type != "rate" {
			return fmt.Errorf("unsupported schedule type: %s", r.Schedule.Type)
		}
	}
	for _, t := range r.Targets {
		if err := t.validate(reg); err != nil {
			return err
		}
		typID := straw.FormatTypeID(t.Type, t.ID)
		rr := reg.Resources[typID]
		if err := r.RequestAccessTo(rr); err != nil {
			return err
		}
	}
	return nil
}

func (t *Target) validate(reg *straw.Registries) error {
	if reg.Services[t.Type] == nil {
		return fmt.Errorf("invalid target type: %s", t.Type)
	}
	// only "func" is supported for now
	// TODO: support SQS, SNS, etc.
	if t.Type != "func" {
		return fmt.Errorf("unsupported target type: %s", t.Type)
	}
	typID := straw.FormatTypeID(t.Type, t.ID)
	if reg.Resources[typID] == nil {
		return fmt.Errorf("unknown %s id: %s", t.Type, t.ID)
	}
	return nil
}

var reName = regexp.MustCompile(`^[\.\-_A-Za-z0-9]+$`)

func validateName(name string) error {
	if !reName.MatchString(name) {
		return fmt.Errorf("invalid event name: %s", name)
	}
	return nil
}

A pkg/service/event/service.go => pkg/service/event/service.go +75 -0
@@ 0,0 1,75 @@
package event

import (
	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/service"
	"git.sr.ht/~mna/straw/pkg/template"
)

var (
	_ straw.Service   = (*Service)(nil)
	_ straw.Generator = (*Service)(nil)
)

var envVarMapping = map[string]string{
	"name": "name",
	"arn":  "arn",
}

// Service is the service implementation for events.
type Service struct {
	RC straw.RunConfig
	CR *straw.CollectedResources
}

// EnvVarMapping returns the mapping of straw attributes to
// terraform attributes.
func (s *Service) EnvVarMapping() map[string]string {
	return envVarMapping
}

// Principal returns the policy principal for the event
// service.
func (s *Service) Principal() *straw.PolicyPrincipal {
	return &straw.PolicyPrincipal{
		Type:        "Service",
		Identifiers: []string{"events.amazonaws.com"},
	}
}

// Scan generates resources for this service by reading the
// provided src directory.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newEvent,
		ApplyFunc:   s.applyEventConfig,
	}
	sc := service.ScanConfig{
		Src:   src,
		Dist:  dist,
		Stage: s.RC.Stage,
	}
	res, err := rs.Scan(sc)
	if len(res) > 0 {
		s.CR.Resources = append(s.CR.Resources, res...)
		s.CR.Generators = append(s.CR.Generators, s)
	}
	return err
}

// Generate implements a straw.Generator for the events.
// It generates the events config.
func (s *Service) Generate(dir string, reg *straw.Registries) error {
	t := template.New("event.tf.tpl").
		FilteredResources("Events", s.CR.Resources, s.Type()).
		ResourcesMap(reg.Resources).
		EnvMap(reg.Environment)

	return t.Execute(dir)
}

// Type returns the type name of this service.
func (s *Service) Type() string {
	return "event"
}

A pkg/service/event/service_test.go => pkg/service/event/service_test.go +97 -0
@@ 0,0 1,97 @@
package event

import (
	"encoding/json"
	"path/filepath"
	"sort"
	"testing"

	"git.sr.ht/~mna/straw"
	"github.com/stretchr/testify/require"
)

func TestService_Scan(t *testing.T) {
	src := filepath.Join("testdata", "src")
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test", ConfigFiles: straw.NewConfigFiles("test")},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)
	require.NoError(t, err)

	expect := []*Resource{
		{
			ID:          "event",
			Name:        "event",
			StagePrefix: true,
			Enabled:     true,
			Schedule: &Schedule{
				Type:  "rate",
				Value: "5 hours",
			},
		},
		{
			ID:          "eventconfig",
			Name:        "ec",
			StagePrefix: true,
			IsPrefix:    true,
			Description: "configured",
			Enabled:     false,
			RoleName:    "arn",
			EventPattern: json.RawMessage(
				`{
  "source": [ "aws.ec2" ],
  "detail-type": [ "EC2 Instance State-change Notification" ],
  "detail": {
    "state": [ "running" ]
  }
}
`,
			),
			Targets: []*Target{
				{TargetID: "1", Type: "func", ID: "fn1"},
				{TargetID: "2", Type: "func", ID: "fn2"},
			},
		},
		{
			ID:          "testonlyevent",
			Name:        "testonlyevent",
			StagePrefix: true,
			Enabled:     true,
			Schedule: &Schedule{
				Type:  "cron",
				Value: "0 12 * * ? *",
			},
		},
	}

	got := make([]*Resource, len(s.CR.Resources))
	for i, r := range s.CR.Resources {
		got[i] = r.(*Resource)
		// nil out the service, unnecessary for the test
		got[i].service = nil
	}
	orderEventsLists(expect, got)
	require.Equal(t, expect, got)
}

func orderEventsLists(events ...[]*Resource) {
	// sort lists to make result deterministic
	for _, v := range events {
		sort.Slice(v, func(i, j int) bool {
			l, r := v[i], v[j]
			return l.ID < r.ID
		})
		for _, vv := range v {
			sort.Slice(vv.Targets, func(i, j int) bool {
				l, r := vv.Targets[i], vv.Targets[j]
				if l.Type == r.Type {
					return l.ID < r.ID
				}
				return l.Type < r.Type
			})
		}
	}
}

A pkg/service/event/testdata/dist/.gitkeep => pkg/service/event/testdata/dist/.gitkeep +0 -0
A pkg/service/event/testdata/src/event/straw.toml => pkg/service/event/testdata/src/event/straw.toml +3 -0
@@ 0,0 1,3 @@
[schedule]
  type = "rate"
  value = "5h"

A pkg/service/event/testdata/src/eventconfig/straw.toml => pkg/service/event/testdata/src/eventconfig/straw.toml +25 -0
@@ 0,0 1,25 @@
name = "ec"
is_prefix = true
description = "configured"
enabled = false
role = "arn"

raw_event_pattern = '''
{
  "source": [ "aws.ec2" ],
  "detail-type": [ "EC2 Instance State-change Notification" ],
  "detail": {
    "state": [ "running" ]
  }
}
'''

[[targets]]
  target_id = "1"
  type = "func"
  id = "fn1"

[[targets]]
  target_id = "2"
  type = "func"
  id = "fn2"

A pkg/service/event/testdata/src/notevent/.gitkeep => pkg/service/event/testdata/src/notevent/.gitkeep +0 -0
A pkg/service/event/testdata/src/prodonlyevent/straw_prod.toml => pkg/service/event/testdata/src/prodonlyevent/straw_prod.toml +0 -0
A pkg/service/event/testdata/src/testonlyevent/straw_test.toml => pkg/service/event/testdata/src/testonlyevent/straw_test.toml +3 -0
@@ 0,0 1,3 @@
[schedule]
  type = "cron"
  value = "0 12 * * ? *"

M pkg/service/fn/config.go => pkg/service/fn/config.go +24 -20
@@ 13,7 13,6 @@ import (
)

type config struct {
	ID                           string `toml:"id"`
	Name                         string `toml:"name"`
	StagePrefix                  bool   `toml:"stage_prefix"`
	Handler                      string `toml:"handler"`


@@ 62,48 61,53 @@ func (s *Service) applyFnConfig(sc service.ScanConfig, res straw.Resource, file 
	}

	fn := res.(*Resource)
	if c.ID != "" {
		fn.ID = c.ID
	}
	if c.Name != "" {
	if meta.IsDefined("name") {
		fn.Name = c.Name
	}
	if meta.IsDefined("stage_prefix") {
		fn.StagePrefix = c.StagePrefix
	}
	if c.Handler != "" {
	if meta.IsDefined("handler") {
		fn.Handler = c.Handler
	}
	if c.Runtime != "" {
	if meta.IsDefined("runtime") {
		fn.Runtime = c.Runtime
	}
	if c.Description != "" {
	if meta.IsDefined("description") {
		fn.Description = c.Description
	}
	if meta.IsDefined("reserved_concurrent_executions") {
		fn.ReservedConcurrentExecutions = c.ReservedConcurrentExecutions
	}
	if c.MemorySize != "" {
		sz, err := units.RAMInBytes(c.MemorySize)
		if err != nil {
			return err
	if meta.IsDefined("memory_size") {
		if c.MemorySize == "" {
			fn.MemorySize = 0
		} else {
			sz, err := units.RAMInBytes(c.MemorySize)
			if err != nil {
				return err
			}
			fn.MemorySize = int(sz / units.MiB)
		}
		fn.MemorySize = int(sz / units.MiB)
	}
	if c.Timeout != "" {
		dur, err := time.ParseDuration(c.Timeout)
		if err != nil {
			return err
	if meta.IsDefined("timeout") {
		if c.Timeout == "" {
			fn.Timeout = 0
		} else {
			dur, err := time.ParseDuration(c.Timeout)
			if err != nil {
				return err
			}
			fn.Timeout = dur
		}
		fn.Timeout = dur
	}
	if c.DeadLetterConfig.Type != "" {
	if meta.IsDefined("dead_letter_config") {
		fn.DeadLetterConfig = &DeadLetterConfig{
			Type: c.DeadLetterConfig.Type,
			ID:   c.DeadLetterConfig.ID,
		}
	}
	if c.Role != "" {
	if meta.IsDefined("role") {
		fn.RoleName = c.Role
	}
	if meta.IsDefined("resources") {

M pkg/service/fn/service.go => pkg/service/fn/service.go +3 -2
@@ 42,8 42,9 @@ func (s *Service) Principal() *straw.PolicyPrincipal {
// build-generated assets.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		NewFunc:   s.newFn,
		ApplyFunc: s.applyFnConfig,
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newFn,
		ApplyFunc:   s.applyFnConfig,
	}
	sc := service.ScanConfig{
		Src:   src,

M pkg/service/fn/service_test.go => pkg/service/fn/service_test.go +3 -3
@@ 15,7 15,7 @@ func TestService_Scan(t *testing.T) {
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test"},
		RC: straw.RunConfig{Stage: "test", ConfigFiles: straw.NewConfigFiles("test")},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)


@@ 31,7 31,7 @@ func TestService_Scan(t *testing.T) {
			Handler:        "index.handler",
		},
		{
			ID:              "fn_test_only",
			ID:              "testonlyfunc",
			Name:            "testonlyfunc",
			StagePrefix:     true,
			PackagedSource:  filepath.Join(dist, "testonlyfunc", "testonlyfunc.zip"),


@@ 40,7 40,7 @@ func TestService_Scan(t *testing.T) {
			ResourceTypeIDs: []string{"queue_q"},
		},
		{
			ID:                           "fn2",
			ID:                           "fnconfig",
			Name:                         "fnname",
			StagePrefix:                  true,
			Description:                  "my func",

M pkg/service/fn/testdata/src/fnconfig/straw.toml => pkg/service/fn/testdata/src/fnconfig/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "fn2"
name = "fnname"
description = "my func"
memory_size = "256M"

M pkg/service/fn/testdata/src/testonlyfunc/straw_test.toml => pkg/service/fn/testdata/src/testonlyfunc/straw_test.toml +0 -1
@@ 1,4 1,3 @@
id = "fn_test_only"

[[resources]]
type = "queue"

M pkg/service/internal/bucket/config.go => pkg/service/internal/bucket/config.go +7 -11
@@ 18,7 18,6 @@ import (

// Config defines the configuration options for buckets.
type Config struct {
	ID              string      `toml:"id"`
	Name            string      `toml:"name"`
	StagePrefix     bool        `toml:"stage_prefix"`
	IsPrefix        bool        `toml:"is_prefix"`


@@ 96,9 95,9 @@ func (c *WebsiteConfig) Apply(b *Bucket, meta *toml.MetaData) error {
		return err
	}

	if c.IndexDocument != "" ||
		c.ErrorDocument != "" ||
		c.RedirectAllRequestsTo != "" {
	if meta.IsDefined("index_document") ||
		meta.IsDefined("error_document") ||
		meta.IsDefined("redirect_all_requests_to") {
		b.Website = &Website{
			IndexDocument:         c.IndexDocument,
			ErrorDocument:         c.ErrorDocument,


@@ 130,10 129,7 @@ func (c *WebsiteConfig) Apply(b *Bucket, meta *toml.MetaData) error {
// Apply applies the configuration c to bucket b, using meta to
// detect if a value was set in the config.
func (c *Config) Apply(b *Bucket, meta *toml.MetaData) error {
	if c.ID != "" {
		b.ID = c.ID
	}
	if c.Name != "" {
	if meta.IsDefined("name") {
		b.Name = c.Name
	}
	if meta.IsDefined("stage_prefix") {


@@ 142,10 138,10 @@ func (c *Config) Apply(b *Bucket, meta *toml.MetaData) error {
	if meta.IsDefined("is_prefix") {
		b.IsPrefix = c.IsPrefix
	}
	if c.ACL != "" {
	if meta.IsDefined("acl") {
		b.ACL = CannedACL(c.ACL)
	}
	if c.Region != "" {
	if meta.IsDefined("region") {
		b.Region = c.Region
	}
	if meta.IsDefined("aes256_encrypted") {


@@ 154,7 150,7 @@ func (c *Config) Apply(b *Bucket, meta *toml.MetaData) error {
	if meta.IsDefined("tags") {
		b.Tags = c.Tags
	}
	if c.CORS != nil {
	if meta.IsDefined("cors") {
		var maxAge time.Duration
		var err error
		if c.CORS.MaxAge != "" {

M pkg/service/queue/config.go => pkg/service/queue/config.go +32 -21
@@ 13,7 13,6 @@ import (
)

type config struct {
	ID                        string            `toml:"id"`
	Name                      string            `toml:"name"`
	StagePrefix               bool              `toml:"stage_prefix"`
	VisibilityTimeout         string            `toml:"visibility_timeout"`


@@ 54,10 53,7 @@ func (s *Service) applyQueueConfig(sc service.ScanConfig, res straw.Resource, fi
	}

	q := res.(*Resource)
	if c.ID != "" {
		q.ID = c.ID
	}
	if c.Name != "" {
	if meta.IsDefined("name") {
		q.Name = c.Name
	}
	q.FIFO = strings.HasSuffix(q.Name, ".fifo")


@@ 68,29 64,44 @@ func (s *Service) applyQueueConfig(sc service.ScanConfig, res straw.Resource, fi
	if meta.IsDefined("content_based_deduplication") {
		q.ContentBasedDeduplication = c.ContentBasedDeduplication
	}
	durations := map[string]*time.Duration{
		c.VisibilityTimeout: &q.VisibilityTimeout,
		c.MessageRetention:  &q.MessageRetention,
		c.Delay:             &q.Delay,
		c.ReceiveWaitTime:   &q.ReceiveWaitTime,

	durations := make(map[string]*time.Duration)
	if meta.IsDefined("visibility_timeout") {
		durations[c.VisibilityTimeout] = &q.VisibilityTimeout
	}
	if meta.IsDefined("message_retention") {
		durations[c.MessageRetention] = &q.MessageRetention
	}
	if meta.IsDefined("delay") {
		durations[c.Delay] = &q.Delay
	}
	if meta.IsDefined("receive_wait_time") {
		durations[c.ReceiveWaitTime] = &q.ReceiveWaitTime
	}
	for k, v := range durations {
		if k != "" {
			dur, err := time.ParseDuration(k)
			if err != nil {
				return err
			}
			*v = dur
		if k == "" {
			*v = 0
			continue
		}
	}
	if c.MaxMessageSize != "" {
		sz, err := units.RAMInBytes(c.MaxMessageSize)
		dur, err := time.ParseDuration(k)
		if err != nil {
			return err
		}
		q.MaxMessageSize = sz
		*v = dur
	}

	if meta.IsDefined("max_message_size") {
		if c.MaxMessageSize == "" {
			q.MaxMessageSize = 0
		} else {
			sz, err := units.RAMInBytes(c.MaxMessageSize)
			if err != nil {
				return err
			}
			q.MaxMessageSize = sz
		}
	}
	if c.Redrive.DeadLetterQueue != "" {
	if meta.IsDefined("redrive") {
		q.Redrive = &RedrivePolicy{
			DeadLetterQueue: c.Redrive.DeadLetterQueue,
			MaxReceive:      c.Redrive.MaxReceive,

M pkg/service/queue/service.go => pkg/service/queue/service.go +3 -2
@@ 43,8 43,9 @@ func (s *Service) Principal() *straw.PolicyPrincipal {
// build-generated assets.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		NewFunc:   s.newQueue,
		ApplyFunc: s.applyQueueConfig,
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newQueue,
		ApplyFunc:   s.applyQueueConfig,
	}
	sc := service.ScanConfig{
		Src:   src,

M pkg/service/queue/service_test.go => pkg/service/queue/service_test.go +3 -3
@@ 15,7 15,7 @@ func TestService_Scan(t *testing.T) {
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test"},
		RC: straw.RunConfig{Stage: "test", ConfigFiles: straw.NewConfigFiles("test")},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)


@@ 27,12 27,12 @@ func TestService_Scan(t *testing.T) {
			Name: "queue",
		},
		{
			ID:          "q_test",
			ID:          "testonlyqueue",
			Name:        "testonlyqueue",
			StagePrefix: true,
		},
		{
			ID:                        "fifoqueue",
			ID:                        "queue.fifo",
			Name:                      "queue.fifo",
			StagePrefix:               true,
			FIFO:                      true,

M pkg/service/queue/testdata/src/queue.fifo/straw.toml => pkg/service/queue/testdata/src/queue.fifo/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "fifoqueue"
content_based_deduplication = true
visibility_timeout = "30s"
message_retention = "45s"

M pkg/service/queue/testdata/src/testonlyqueue/straw_test.toml => pkg/service/queue/testdata/src/testonlyqueue/straw_test.toml +0 -1
@@ 1,1 0,0 @@
id = "q_test"

M pkg/service/service.go => pkg/service/service.go +14 -19
@@ 1,12 1,11 @@
package service

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"git.sr.ht/~mna/straw"
	"git.sr.ht/~mna/straw/pkg/errors"
)

// ScanConfig holds the configuration for a Scan of a


@@ 21,6 20,10 @@ type ScanConfig struct {
// a service and create new resources with configuration files
// properly applied.
type ResourceScanner struct {
	// ConfigFiles is the list of relevant configuration files.
	// Typically set from the RunConfig struct.
	ConfigFiles straw.ConfigFiles

	// NewFunc is called for each subdirectory in a service's
	// directory. It creates the resource with default values.
	// The resource may be later discarded if there was no


@@ 65,28 68,20 @@ func (s *ResourceScanner) Scan(base ScanConfig) ([]straw.Resource, error) {
		// create the resource with default values
		cur := s.NewFunc(dirConf)

		var configured bool
		files := []string{
			filepath.Join(dirConf.Src, straw.ResourceConfigFilename),
			filepath.Join(dirConf.Src, fmt.Sprintf(straw.FmtStageResourceConfigFilename, dirConf.Stage)),
		}

		// apply the configuration files in layers
		for _, file := range files {
			if err := s.ApplyFunc(dirConf, cur, file); err != nil {
				if os.IsNotExist(err) {
					continue
				}
				return res, err
			}
			configured = true
		count, err := s.ConfigFiles.EachIn(dirConf.Src, func(file string) error {
			return s.ApplyFunc(dirConf, cur, file)
		})
		if err != nil {
			typ, id := cur.TypeID()
			return res, &errors.ResourceError{Err: err, Type: typ, ID: id}
		}

		// keep the resource only if there was at least one config file.
		if configured {
		if count > 0 {
			if s.PostFunc != nil {
				if err := s.PostFunc(dirConf, cur); err != nil {
					return res, err
					typ, id := cur.TypeID()
					return res, &errors.ResourceError{Err: err, Type: typ, ID: id}
				}
			}
			res = append(res, cur)

M pkg/service/service_test.go => pkg/service/service_test.go +2 -0
@@ 17,8 17,10 @@ func TestResourceScanner_Scan(t *testing.T) {
		Src:   "testdata/scan",
		Dist:  "testdata/scan",
	}
	cf := straw.NewConfigFiles("test")

	rs := service.ResourceScanner{
		ConfigFiles: cf,
		NewFunc: func(sc service.ScanConfig) straw.Resource {
			base := filepath.Base(sc.Src)
			return &testing.MockResource{

M pkg/service/site/service.go => pkg/service/site/service.go +4 -3
@@ 44,9 44,10 @@ func (s *Service) Principal() *straw.PolicyPrincipal {
// build-generated assets.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		NewFunc:   s.newSite,
		ApplyFunc: s.applySiteConfig,
		PostFunc:  s.postSite,
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newSite,
		ApplyFunc:   s.applySiteConfig,
		PostFunc:    s.postSite,
	}
	sc := service.ScanConfig{
		Src:   src,

M pkg/service/site/service_test.go => pkg/service/site/service_test.go +3 -3
@@ 20,7 20,7 @@ func TestService_Scan(t *testing.T) {
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test"},
		RC: straw.RunConfig{Stage: "test", ConfigFiles: straw.NewConfigFiles("test")},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)


@@ 29,7 29,7 @@ func TestService_Scan(t *testing.T) {
	expect := []*Resource{
		{
			Bucket: &bucket.Bucket{
				ID:          "site_test",
				ID:          "testonlysite",
				Name:        "testonlysite",
				StagePrefix: true,
				ACL:         bucket.Private,


@@ 63,7 63,7 @@ func TestService_Scan(t *testing.T) {
		},
		{
			Bucket: &bucket.Bucket{
				ID:          "site.com",
				ID:          "config.site.com",
				Name:        "config.site.com",
				StagePrefix: true,
				ACL:         bucket.PublicRead,

M pkg/service/site/testdata/src/config.site.com/straw.toml => pkg/service/site/testdata/src/config.site.com/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "site.com"
acl = "public-read"
index_document = "index.html"


M pkg/service/site/testdata/src/testonlysite/straw_test.toml => pkg/service/site/testdata/src/testonlysite/straw_test.toml +0 -1
@@ 1,1 0,0 @@
id = "site_test"

M pkg/service/storage/service.go => pkg/service/storage/service.go +4 -3
@@ 43,9 43,10 @@ func (s *Service) Principal() *straw.PolicyPrincipal {
// build-generated assets.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		NewFunc:   s.newStorage,
		ApplyFunc: s.applyStorageConfig,
		PostFunc:  s.postStorage,
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newStorage,
		ApplyFunc:   s.applyStorageConfig,
		PostFunc:    s.postStorage,
	}
	sc := service.ScanConfig{
		Src:   src,

M pkg/service/storage/service_test.go => pkg/service/storage/service_test.go +3 -3
@@ 19,7 19,7 @@ func TestService_Scan(t *testing.T) {
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test"},
		RC: straw.RunConfig{Stage: "test", ConfigFiles: straw.NewConfigFiles("test")},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)


@@ 28,7 28,7 @@ func TestService_Scan(t *testing.T) {
	expect := []*Resource{
		{
			Bucket: &bucket.Bucket{
				ID:          "with-contents",
				ID:          "bucketcontents",
				Name:        "bucketcontents",
				StagePrefix: true,
				ACL:         "private",


@@ 73,7 73,7 @@ func TestService_Scan(t *testing.T) {
		},
		{
			Bucket: &bucket.Bucket{
				ID:          "bucket_test",
				ID:          "testonlybucket",
				Name:        "testonlybucket",
				StagePrefix: true,
				ACL:         "private",

M pkg/service/storage/testdata/src/bucketcontents/straw.toml => pkg/service/storage/testdata/src/bucketcontents/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "with-contents"

[[notifications]]
  type = "queue"

M pkg/service/storage/testdata/src/testonlybucket/straw_test.toml => pkg/service/storage/testdata/src/testonlybucket/straw_test.toml +0 -1
@@ 1,1 0,0 @@
id = "bucket_test"

M pkg/service/topic/config.go => pkg/service/topic/config.go +6 -7
@@ 2,6 2,7 @@ package topic

import (
	"encoding/json"
	"fmt"
	"path/filepath"

	"git.sr.ht/~mna/straw"


@@ 10,7 11,6 @@ import (
)

type config struct {
	ID            string `toml:"id"`
	Name          string `toml:"name"`
	StagePrefix   bool   `toml:"stage_prefix"`
	DisplayName   string `toml:"display_name"`


@@ 40,16 40,13 @@ func (s *Service) applyTopicConfig(sc service.ScanConfig, res straw.Resource, fi
	}

	t := res.(*Resource)
	if c.ID != "" {
		t.ID = c.ID
	}
	if c.Name != "" {
	if meta.IsDefined("name") {
		t.Name = c.Name
	}
	if meta.IsDefined("stage_prefix") {
		t.StagePrefix = c.StagePrefix
	}
	if c.DisplayName != "" {
	if meta.IsDefined("display_name") {
		t.DisplayName = c.DisplayName
	}
	if meta.IsDefined("subscriptions") {


@@ 59,8 56,10 @@ func (s *Service) applyTopicConfig(sc service.ScanConfig, res straw.Resource, fi
				Type: s.Type,
				ID:   s.ID,
			}
			// TODO: validate raw filter policy is valid JSON
			if s.RawFilterPolicy != "" {
				if !json.Valid([]byte(s.RawFilterPolicy)) {
					return fmt.Errorf("invalid JSON in %s.%s.raw_filter_policy subscription configuration", s.Type, s.ID)
				}
				sub.FilterPolicy = json.RawMessage(s.RawFilterPolicy)
			}
			t.Subscriptions[i] = sub

M pkg/service/topic/service.go => pkg/service/topic/service.go +3 -2
@@ 42,8 42,9 @@ func (s *Service) Principal() *straw.PolicyPrincipal {
// build-generated assets.
func (s *Service) Scan(src, dist string) error {
	rs := &service.ResourceScanner{
		NewFunc:   s.newTopic,
		ApplyFunc: s.applyTopicConfig,
		ConfigFiles: s.RC.ConfigFiles,
		NewFunc:     s.newTopic,
		ApplyFunc:   s.applyTopicConfig,
	}
	sc := service.ScanConfig{
		Src:   src,

M pkg/service/topic/service_test.go => pkg/service/topic/service_test.go +2 -2
@@ 15,7 15,7 @@ func TestService_Scan(t *testing.T) {
	dist := filepath.Join("testdata", "dist")

	s := &Service{
		RC: straw.RunConfig{Stage: "test"},
		RC: straw.RunConfig{Stage: "test", ConfigFiles: straw.NewConfigFiles("test")},
		CR: &straw.CollectedResources{},
	}
	err := s.Scan(src, dist)


@@ 33,7 33,7 @@ func TestService_Scan(t *testing.T) {
			StagePrefix: false,
		},
		{
			ID:          "topic2",
			ID:          "topic",
			Name:        "topic",
			StagePrefix: true,
			DisplayName: "my-topic",

M pkg/service/topic/testdata/src/topic/straw.toml => pkg/service/topic/testdata/src/topic/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "topic2"
display_name = "my-topic"

[[subscriptions]]

M pkg/template/internal/esctpl/gen_templates.go => pkg/template/internal/esctpl/gen_templates.go +32 -15
@@ 212,22 212,22 @@ var _escData = map[string]*_escFile{
	"/templates/api.tf.tpl": {
		name:    "api.tf.tpl",
		local:   "templates/api.tf.tpl",
		size:    2016,
		modtime: 1546451402,
		size:    2227,
		modtime: 1547154971,
		compressed: `
H4sIAAAAAAAC/9RVTYsjNxC9968omsllybZ3s+RimMMQG7KHGZuMIQkhiNrusl2hW1IktY1H6L8Hqb9s
z8SbQC45uVVSPb169VT23qDcEdyhZpjfQ/Gg2YaQGbKqNSVBjkcrULPYoaMjnoQh62Igh9z74vMihBx8
BiCxIbhPweV2S6XjAz1hQyHkGYD3vIViQbY0rB0rGUI1LeAevNeGpdtC/s2f+dVJ70lWIYwwjyx/UI02
ZC0r+cwvFELDkpu2EeW0ISy/UIJ+M2NAHWGXstKKpducNKXbqA+IUskt71qDiWysFsCdNFm4h99SxRep
+e8ZQEf36oq1qrk8haDTb5TrzlfosIgqMzai2xCVKtuGpCu873OS0sUfVsmQD7Ahy7w/sttPwJn3jhpd
o+tb9wZoDkU62HMbLaDR7ZMH1uj2XzVB2rk2wWAOwVVX3N+5p/A+Wq6riqszj6zRkHQxngHotLiFlmgk
mca8CY9qS/8Q5zUro5QbbxATZm/FqJbQaFxv+jUaF71+pmjxSG6vKpvUvmvSImr7eRFlTn4fzkAu8q4D
6fIb0ncwSfgLyP+kBWflflXxie2QvXdO9/x6UbriuhGArdsrwy/Yv/j8afW0zMH72TvYrBareToB72aD
s6MbfkRZ1WRuKsLS0a57m/8HWV5nd3vFK+bFWWZCOqtUXKGuV8+beCSOpbh++PlZrH9a/fJrDLaGYwyN
nOPRzlFzf/X8zh/QFIZ2cdDOa2y+VDiPFcy++/Dx+/cfPr3/9HG2bWUZ77Qz753akDG4VaZ5MHJsUBEn
37SK7GcsD6pMZO30vMfz3JBqXQiu+xAN1zVbKpWsbBraTj32IftG2jQDh6/r3xuOqUjX6tTNwssBdubG
ijTJygolAev6XHvbmfTfmerCEtbhjsTwl9k1geSBjZKRVT9qRioHNIxfarLfguWGazTgFCynBGAJsUt2
eD6DCH8FAAD//wPkxaTgBwAA
H4sIAAAAAAAC/6xVTW8jNwy9+1cQQnpZLCb3BXIImgDdQ2KjCdAWi4WgztA22xlJK3EcOIL+eyFpPmzX
6+wCOc2IQz6Rj4+cEJzSG4QrZQk+3UB1a8nHuHDoTe9qBKFevFSW5EYxvqi9dOg5GQSIEKrPdzEKCAsA
rTqEm2y8X6+xZtrho+owRrEACIHWUN2hrx1ZJqNjbOYD3EAI1pHmNYhfvokTzxBQNzFOMA+kfzWddeg9
Gf1ErxhjR5q6vpP1/EF6esUMfTZiRJ1g73VjDWl+3lvMt+FgkLXRa9r0TuVkU7UAvLfo4Qa+5IqPQsXX
BUBJ9+SKlWmp3sdo8zPRdRUaxapKLJPqZPkgG1P3HWquQhhiMtPVP97oKEbYuFiE8EK8nYEXITB2tlU8
tO4MqIAqOw65TRKwirdZAyvFW59daA0ak8ExiGvxljDyl1NhjIKR1JSCv6eoKoQkw1IpNQe6WSmHmpN9
AWDz4RJaTiNTN8XNeNh6/EGc/2fljOHpBjljDvJMDEqbyCqDkHhL+j/DdvWAvDXN5Vnrss/7E4rfSrcP
G5txx7regZKR5h9BHRtWkjrp18DtltkOhAzsFgbLflE9b42jVzWsE/G4fLwXEML1B3he3i0/ZQ/4cD2O
TZLVb0o3LbqLLSDNuCmD/659eJOWUmo13FidMnxCx5vxB/45/qAueYK1Wj49J5e04dL59o8nufp9+edf
ydg7KvSzeUbn1Nq47pbZTWxWaQfOp893IEjvzL8oldPi8G8wBVCHpucYubzIjtqWPNZGNz7vbzYPg8mf
CZvX4fh2+rzQ3wZta/ZlLR63t0GLuvEy6+lL3vnfXZVnphqgRBwJ7ZK8xlaJj7PopwGYTV9/UnlHqvOs
NijHX/VV2ClXod6RMzpRMDRnmpmdcqT+btF/BE8dtcoBG7ifA4A0rHtd+2myTM+2ZxApm+Fy2bu2ULpT
bY/nM577MOdcZNO79miF/hcAAP//4mFUvbMIAAA=
`,
	},



@@ 287,6 287,22 @@ RJctPpovE7OcbyKR3XM1ub1QFB3T0RpkUPrUq9/2mm3XbxfRzyH+WOzpzW28b3XMtut7DaG4qr4CAAD/
`,
	},

	"/templates/event.tf.tpl": {
		name:    "event.tf.tpl",
		local:   "templates/event.tf.tpl",
		size:    799,
		modtime: 1547413263,
		compressed: `
H4sIAAAAAAAC/4xST+/TMAy951NY0ZDgQD8A0g5Ivx3G4cfEJq5RaN0tUpYUJ+2GIn931KR/YAzpd2rt
vGc/2y8l0u6MsMEBPm2h2g3oYmAWhMH3VCNIfQuqtr5vbjrWF4UjQlFvUYJMqdq/MEtIAiAl00K1DwfC
1tyZBYDTV1RdjmGb4bu2xTqaAV/1FZk/ykxEG3AhPEdOQNdkXOn1gqEm00XjHXOzBrCFlDoyLrYg3/2U
D8jHMt+8nZqQt6g0uUnC+iBXjgkKnf5hsfm7TZRQ7coDs1iKH+sLNr0t44UpUHjvCEMoWsdWM6w6/eqQ
+f2fqe/a9sj8Yd0VjJXzrQ46RiSXq5fTdCWTxUW/d9Y4/HL8+vqEMA/FQixGiJrOGLMZTvn3DW4onOyH
DQ7ZEiqlaRT1xCSl8pgtVGWaf2+2gtblj74bN7ZJ/7VltYqoRjtl5yw3jf6ERLr1dP1MDrJGKPoEi7nR
/P0dAAD//5ojlQkfAwAA
`,
	},

	"/templates/func.tf.tpl": {
		name:    "func.tf.tpl",
		local:   "templates/func.tf.tpl",


@@ 409,6 425,7 @@ var _escDirs = map[string][]os.FileInfo{
		_escData["/templates/bootstrap.tf.tpl"],
		_escData["/templates/bucket.tf.tpl"],
		_escData["/templates/core.tf.tpl"],
		_escData["/templates/event.tf.tpl"],
		_escData["/templates/func.tf.tpl"],
		_escData["/templates/local.tf.tpl"],
		_escData["/templates/partials"],

M pkg/testdata/02-bucket-base-props/src/storage/a/straw.toml => pkg/testdata/02-bucket-base-props/src/storage/a/straw.toml +0 -1
@@ 1,4 1,3 @@
id = "a"
name = "ab"
is_prefix = true
acl = "public-read"

A pkg/testdata/23-api-minimal/dist/.gitkeep => pkg/testdata/23-api-minimal/dist/.gitkeep +0 -0
A pkg/testdata/23-api-minimal/golden.json => pkg/testdata/23-api-minimal/golden.json +82 -0
@@ 0,0 1,82 @@
{
  "api_a": {
    "ID": "a",
    "Name": "a",
    "StagePrefix": true,
    "Description": "",
    "EndpointType": "REGIONAL",
    "MinCompressionSize": 0,
    "Policy": null,
    "Paths": [
      {
        "ID": "a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "ParentID": "",
        "Part": "base",
        "Methods": [
          {
            "ID": "ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
            "Method": "ANY",
            "Auth": null,
            "Handler": null
          }
        ]
      },
      {
        "ID": "a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
        "ParentID": "a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "Part": "one",
        "Methods": [
          {
            "ID": "GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
            "Method": "GET",
            "Auth": null,
            "Handler": null
          }
        ]
      },
      {
        "ID": "a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
        "ParentID": "a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "Part": "two",
        "Methods": [
          {
            "ID": "GET_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
            "Method": "GET",
            "Auth": null,
            "Handler": null
          },
          {
            "ID": "POST_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
            "Method": "POST",
            "Auth": null,
            "Handler": null
          }
        ]
      }
    ]
  },
  "apipathmethod_ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM": {
    "ID": "ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
    "Method": "ANY",
    "Auth": null,
    "Handler": null
  },
  "apipathmethod_GET_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo": {
    "ID": "GET_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
    "Method": "GET",
    "Auth": null,
    "Handler": null
  },
  "apipathmethod_GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE": {
    "ID": "GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
    "Method": "GET",
    "Auth": null,
    "Handler": null
  },
  "apipathmethod_POST_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo": {
    "ID": "POST_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
    "Method": "POST",
    "Auth": null,
    "Handler": null
  }
}
\ No newline at end of file

A pkg/testdata/23-api-minimal/src/api/a/base/one/straw.toml => pkg/testdata/23-api-minimal/src/api/a/base/one/straw.toml +2 -0
@@ 0,0 1,2 @@
[[methods]]
  method = 'GET'

A pkg/testdata/23-api-minimal/src/api/a/base/straw.toml => pkg/testdata/23-api-minimal/src/api/a/base/straw.toml +2 -0
@@ 0,0 1,2 @@
[[methods]]
  method = 'ANY'

A pkg/testdata/23-api-minimal/src/api/a/base/two/straw.toml => pkg/testdata/23-api-minimal/src/api/a/base/two/straw.toml +5 -0
@@ 0,0 1,5 @@
[[methods]]
  method = 'GET'

[[methods]]
  method = 'POST'

A pkg/testdata/23-api-minimal/src/api/a/notapath/.gitkeep => pkg/testdata/23-api-minimal/src/api/a/notapath/.gitkeep +0 -0
A pkg/testdata/23-api-minimal/src/api/a/straw.toml => pkg/testdata/23-api-minimal/src/api/a/straw.toml +1 -0
@@ 0,0 1,1 @@
endpoint_type = "REGIONAL"

A pkg/testdata/23-api-minimal/tmp/.gitignore => pkg/testdata/23-api-minimal/tmp/.gitignore +2 -0
@@ 0,0 1,2 @@
*
!.gitignore

A pkg/testdata/24-api-root-handler/dist/func/fn/fn.zip => pkg/testdata/24-api-root-handler/dist/func/fn/fn.zip +0 -0
A pkg/testdata/24-api-root-handler/golden.json => pkg/testdata/24-api-root-handler/golden.json +126 -0
@@ 0,0 1,126 @@
{
  "api_a": {
    "ID": "a",
    "Name": "a",
    "StagePrefix": true,
    "Description": "api a",
    "EndpointType": "REGIONAL",
    "MinCompressionSize": 1024,
    "Policy": null,
    "Paths": [
      {
        "ID": "a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
        "ParentID": "",
        "Part": "/",
        "Methods": [
          {
            "ID": "GET_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
            "Method": "GET",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn",
              "Timeout": 10000000000
            }
          },
          {
            "ID": "HEAD_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
            "Method": "HEAD",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn",
              "Timeout": 10000000000
            }
          }
        ]
      },
      {
        "ID": "a_VIqi9GY16V4r40ZVFKuC0RJDbxJFufPtyOaEpzxa1HU",
        "ParentID": "",
        "Part": "{proxy+}",
        "Methods": [
          {
            "ID": "ANY_a_VIqi9GY16V4r40ZVFKuC0RJDbxJFufPtyOaEpzxa1HU",
            "Method": "ANY",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn",
              "Timeout": 20000000000
            }
          }
        ]
      }
    ]
  },
  "apipathmethod_ANY_a_VIqi9GY16V4r40ZVFKuC0RJDbxJFufPtyOaEpzxa1HU": {
    "ID": "ANY_a_VIqi9GY16V4r40ZVFKuC0RJDbxJFufPtyOaEpzxa1HU",
    "Method": "ANY",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn",
      "Timeout": 20000000000
    }
  },
  "apipathmethod_GET_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU": {
    "ID": "GET_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
    "Method": "GET",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn",
      "Timeout": 10000000000
    }
  },
  "apipathmethod_HEAD_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU": {
    "ID": "HEAD_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
    "Method": "HEAD",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn",
      "Timeout": 10000000000
    }
  },
  "func_fn": {
    "ID": "fn",
    "Name": "fn",
    "StagePrefix": true,
    "PackagedSource": "/testdata/24-api-root-handler/dist/func/fn/fn.zip",
    "Handler": "main",
    "Description": "",
    "MemorySize": 0,
    "Runtime": "go1.x",
    "Timeout": 0,
    "ReservedConcurrentExecutions": 0,
    "Publish": false,
    "Tags": null,
    "Environment": null,
    "DeadLetterConfig": null,
    "RoleName": "",
    "Resources": null,
    "ResourceTypeIDs": null,
    "Permissions": [
      {
        "ID": "fn_apipathmethod_GET_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
        "SourceType": "apipathmethod",
        "SourceID": "GET_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
        "Principal": "apigateway.amazonaws.com"
      },
      {
        "ID": "fn_apipathmethod_HEAD_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
        "SourceType": "apipathmethod",
        "SourceID": "HEAD_a_47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU",
        "Principal": "apigateway.amazonaws.com"
      },
      {
        "ID": "fn_apipathmethod_ANY_a_VIqi9GY16V4r40ZVFKuC0RJDbxJFufPtyOaEpzxa1HU",
        "SourceType": "apipathmethod",
        "SourceID": "ANY_a_VIqi9GY16V4r40ZVFKuC0RJDbxJFufPtyOaEpzxa1HU",
        "Principal": "apigateway.amazonaws.com"
      }
    ]
  }
}
\ No newline at end of file

A pkg/testdata/24-api-root-handler/src/api/a/straw.toml => pkg/testdata/24-api-root-handler/src/api/a/straw.toml +16 -0
@@ 0,0 1,16 @@
description = "api a"
endpoint_type = "REGIONAL"
minimum_compression_size = "1k"

[root]
  [[root.methods]]
    method = "GET"
    handler_type = "func"
    handler_id = "fn"
    timeout = "10s"

  [[root.methods]]
    method = "HEAD"
    handler_type = "func"
    handler_id = "fn"
    timeout = "10s"

A pkg/testdata/24-api-root-handler/src/api/a/{proxy+}/straw.toml => pkg/testdata/24-api-root-handler/src/api/a/{proxy+}/straw.toml +5 -0
@@ 0,0 1,5 @@
[[methods]]
  method = "ANY"
  handler_type = "func"
  handler_id = "fn"
  timeout = "20s"

A pkg/testdata/24-api-root-handler/src/func/fn/main.go => pkg/testdata/24-api-root-handler/src/func/fn/main.go +4 -0
@@ 0,0 1,4 @@
package main

func main() {
}

A pkg/testdata/24-api-root-handler/src/func/fn/straw.toml => pkg/testdata/24-api-root-handler/src/func/fn/straw.toml +0 -0
A pkg/testdata/24-api-root-handler/tmp/.gitignore => pkg/testdata/24-api-root-handler/tmp/.gitignore +2 -0
@@ 0,0 1,2 @@
*
!.gitignore

A pkg/testdata/25-api-many-handlers/dist/func/fn1/fn1.zip => pkg/testdata/25-api-many-handlers/dist/func/fn1/fn1.zip +0 -0
A pkg/testdata/25-api-many-handlers/dist/func/fn2/fn2.zip => pkg/testdata/25-api-many-handlers/dist/func/fn2/fn2.zip +0 -0
A pkg/testdata/25-api-many-handlers/dist/func/fn3/fn3.zip => pkg/testdata/25-api-many-handlers/dist/func/fn3/fn3.zip +0 -0
A pkg/testdata/25-api-many-handlers/golden.json => pkg/testdata/25-api-many-handlers/golden.json +201 -0
@@ 0,0 1,201 @@
{
  "api_a": {
    "ID": "a",
    "Name": "a",
    "StagePrefix": true,
    "Description": "",
    "EndpointType": "REGIONAL",
    "MinCompressionSize": 0,
    "Policy": null,
    "Paths": [
      {
        "ID": "a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "ParentID": "",
        "Part": "base_resource",
        "Methods": [
          {
            "ID": "ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
            "Method": "ANY",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn1",
              "Timeout": 0
            }
          }
        ]
      },
      {
        "ID": "a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
        "ParentID": "a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "Part": "one",
        "Methods": [
          {
            "ID": "GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
            "Method": "GET",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn2",
              "Timeout": 0
            }
          },
          {
            "ID": "POST_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
            "Method": "POST",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn3",
              "Timeout": 0
            }
          }
        ]
      },
      {
        "ID": "a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
        "ParentID": "a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "Part": "two",
        "Methods": [
          {
            "ID": "ANY_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
            "Method": "ANY",
            "Auth": null,
            "Handler": {
              "Type": "func",
              "ID": "fn3",
              "Timeout": 0
            }
          }
        ]
      }
    ]
  },
  "apipathmethod_ANY_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo": {
    "ID": "ANY_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
    "Method": "ANY",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn3",
      "Timeout": 0
    }
  },
  "apipathmethod_ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM": {
    "ID": "ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
    "Method": "ANY",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn1",
      "Timeout": 0
    }
  },
  "apipathmethod_GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE": {
    "ID": "GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
    "Method": "GET",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn2",
      "Timeout": 0
    }
  },
  "apipathmethod_POST_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE": {
    "ID": "POST_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
    "Method": "POST",
    "Auth": null,
    "Handler": {
      "Type": "func",
      "ID": "fn3",
      "Timeout": 0
    }
  },
  "func_fn1": {
    "ID": "fn1",
    "Name": "fn1",
    "StagePrefix": true,
    "PackagedSource": "/testdata/25-api-many-handlers/dist/func/fn1/fn1.zip",
    "Handler": "main",
    "Description": "",
    "MemorySize": 0,
    "Runtime": "go1.x",
    "Timeout": 0,
    "ReservedConcurrentExecutions": 0,
    "Publish": false,
    "Tags": null,
    "Environment": null,
    "DeadLetterConfig": null,
    "RoleName": "",
    "Resources": null,
    "ResourceTypeIDs": null,
    "Permissions": [
      {
        "ID": "fn1_apipathmethod_ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "SourceType": "apipathmethod",
        "SourceID": "ANY_a_fP4znqSkvvyVABf_ScHegby2QUU3PTxK8lGqSZ1hBNM",
        "Principal": "apigateway.amazonaws.com"
      }
    ]
  },
  "func_fn2": {
    "ID": "fn2",
    "Name": "fn2",
    "StagePrefix": true,
    "PackagedSource": "/testdata/25-api-many-handlers/dist/func/fn2/fn2.zip",
    "Handler": "main",
    "Description": "",
    "MemorySize": 0,
    "Runtime": "go1.x",
    "Timeout": 0,
    "ReservedConcurrentExecutions": 0,
    "Publish": false,
    "Tags": null,
    "Environment": null,
    "DeadLetterConfig": null,
    "RoleName": "",
    "Resources": null,
    "ResourceTypeIDs": null,
    "Permissions": [
      {
        "ID": "fn2_apipathmethod_GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
        "SourceType": "apipathmethod",
        "SourceID": "GET_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
        "Principal": "apigateway.amazonaws.com"
      }
    ]
  },
  "func_fn3": {
    "ID": "fn3",
    "Name": "fn3",
    "StagePrefix": true,
    "PackagedSource": "/testdata/25-api-many-handlers/dist/func/fn3/fn3.zip",
    "Handler": "main",
    "Description": "",
    "MemorySize": 0,
    "Runtime": "go1.x",
    "Timeout": 0,
    "ReservedConcurrentExecutions": 0,
    "Publish": false,
    "Tags": null,
    "Environment": null,
    "DeadLetterConfig": null,
    "RoleName": "",
    "Resources": null,
    "ResourceTypeIDs": null,
    "Permissions": [
      {
        "ID": "fn3_apipathmethod_POST_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
        "SourceType": "apipathmethod",
        "SourceID": "POST_a_zwFatUWyQgLNtHOIZoMk6hqSYGl9NvTlQc8BntMF0PE",
        "Principal": "apigateway.amazonaws.com"
      },
      {
        "ID": "fn3_apipathmethod_ANY_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
        "SourceType": "apipathmethod",
        "SourceID": "ANY_a_Zz-2NpC6nMxNnhFIoO1LcWNnwxK5VHA8JkeXJuIvylo",
        "Principal": "apigateway.amazonaws.com"
      }
    ]
  }
}
\ No newline at end of file

A pkg/testdata/25-api-many-handlers/src/api/a/base/one/straw.toml => pkg/testdata/25-api-many-handlers/src/api/a/base/one/straw.toml +9 -0
@@ 0,0 1,9 @@
[[methods]]
  method = "GET"
  handler_type = "func"
  handler_id = "fn2"

[[methods]]
  method = "POST"
  handler_type = "func"
  handler_id = "fn3"

A pkg/testdata/25-api-many-handlers/src/api/a/base/straw.toml => pkg/testdata/25-api-many-handlers/src/api/a/base/straw.toml +6 -0
@@ 0,0 1,6 @@
part = "base_resource"

[[methods]]
  method = "ANY"
  handler_type = "func"
  handler_id = "fn1"

A pkg/testdata/25-api-many-handlers/src/api/a/base/two/straw.toml => pkg/testdata/25-api-many-handlers/src/api/a/base/two/straw.toml +4 -0
@@ 0,0 1,4 @@
[[methods]]
  method = "ANY"
  handler_type = "func"
  handler_id = "fn3"

A pkg/testdata/25-api-many-handlers/src/api/a/straw.toml => pkg/testdata/25-api-many-handlers/src/api/a/straw.toml +1 -0
@@ 0,0 1,1 @@
endpoint_type = "REGIONAL"

A pkg/testdata/25-api-many-handlers/src/func/fn1/main.go => pkg/testdata/25-api-many-handlers/src/func/fn1/main.go +4 -0
@@ 0,0 1,4 @@
package main

func main() {
}

A pkg/testdata/25-api-many-handlers/src/func/fn1/straw.toml => pkg/testdata/25-api-many-handlers/src/func/fn1/straw.toml +0 -0
A pkg/testdata/25-api-many-handlers/src/func/fn2/main.go => pkg/testdata/25-api-many-handlers/src/func/fn2/main.go +4 -0
@@ 0,0 1,4 @@
package main

func main() {
}

A pkg/testdata/25-api-many-handlers/src/func/fn2/straw.toml => pkg/testdata/25-api-many-handlers/src/func/fn2/straw.toml +0 -0
A pkg/testdata/25-api-many-handlers/src/func/fn3/main.go => pkg/testdata/25-api-many-handlers/src/func/fn3/main.go +4 -0
@@ 0,0 1,4 @@
package main

func main() {
}

A pkg/testdata/25-api-many-handlers/src/func/fn3/straw.toml => pkg/testdata/25-api-many-handlers/src/func/fn3/straw.toml +0 -0
A pkg/testdata/25-api-many-handlers/tmp/.gitignore => pkg/testdata/25-api-many-handlers/tmp/.gitignore +2 -0
@@ 0,0 1,2 @@
*
!.gitignore

A pkg/testdata/28-terraform-copy/dist/.gitkeep => pkg/testdata/28-terraform-copy/dist/.gitkeep +0 -0
A pkg/testdata/28-terraform-copy/golden.json => pkg/testdata/28-terraform-copy/golden.json +1 -0