~nilium/sql

6dc63361c7e03f9feaa77fa2daace76e816c6eda — Noel Cower 10 months ago 1154554 main v0.3.0
Add jq::expr argument support

Add initial jq::expr argument support. This is still sort of
experimental, so may break later if I find a better way to handle this
sort of thing. For now, this provides a convenient way to chain results
together so that if one query produces a value the next query needs,
that can be used.

The man page has been updated accordingly for this.
4 files changed, 144 insertions(+), 22 deletions(-)

M go.mod
M go.sum
M main.go
M sql.1.scd
M go.mod => go.mod +3 -0
@@ 6,8 6,11 @@ require (
	github.com/adrg/xdg v0.3.0
	github.com/go-sql-driver/mysql v1.5.0
	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
	github.com/itchyny/gojq v0.12.4
	github.com/jmoiron/sqlx v1.3.1
	github.com/lib/pq v1.8.0
	github.com/mattn/go-sqlite3 v1.14.8
	go.spiff.io/go-ini v0.2.0
)

require github.com/itchyny/timefmt-go v0.1.3 // indirect

M go.sum => go.sum +14 -1
@@ 4,13 4,21 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA=
github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o=
github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg=
github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE=
github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=


@@ 21,6 29,11 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
go.spiff.io/go-ini v0.2.0 h1:JgkxbSQ1ZjBrfQcBu3EmdDBJmY+U6E2IuTtP9nTpCXM=
go.spiff.io/go-ini v0.2.0/go.mod h1:h4CttGw62dZCr2s9w4nlYF2jl4jJei7/yUfBg9ieeng=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

M main.go => main.go +76 -3
@@ 28,6 28,7 @@ import (
	"io"
	"io/ioutil"
	"log"
	"math/big"
	"net/url"
	"os"
	"path/filepath"


@@ 36,6 37,7 @@ import (

	"github.com/adrg/xdg"
	"github.com/google/shlex"
	"github.com/itchyny/gojq"
	"go.spiff.io/go-ini"
	"go.spiff.io/sql/driver"
	"go.spiff.io/sql/internal/cli"


@@ 528,14 530,33 @@ splitArgs:
	_ = ctx
	_ = opts
	err = func() (err error) {
		var last []interface{}
		var allData = make([]interface{}, 0, len(queries))

		defer done(&err)
		for i, query := range queries {
			i := i + 1

			// Expand any jq arguments found in the argument sets.
			for i, argset := range query.Args {
				argset, err := expandArgs(ctx, argset, allData, last, 0)
				if err != nil {
					return fmt.Errorf("error expanding arguments: %w", err)
				}
				query.Args[i] = argset
				for i, arg := range argset {
					argset[i] = flatten(arg, false)
				}
			}

			results, err := query.Exec(ctx, conn)
			if err != nil {
				return fmt.Errorf("error executing query %d: %w", i, err)
			}

			last = results.Opaque()
			allData = append(allData, last)

			for ri, recs := range results {
				if printArray {
					if err = enc.Encode(recs); err != nil {


@@ 618,7 639,7 @@ func parseQueryArgs(sets [][]string) ([][]interface{}, error) {
		}

		for i, arg := range root.args {
			root.args[i] = flatten(arg)
			root.args[i] = flatten(arg, false)
		}

		qargs[i] = root.args


@@ 627,12 648,16 @@ func parseQueryArgs(sets [][]string) ([][]interface{}, error) {
	return qargs, nil
}

func flatten(v interface{}) interface{} {
func flatten(v interface{}, marshal bool) interface{} {
	switch v := v.(type) {
	case []interface{}:
		if marshal {
			b, _ := json.Marshal(v)
			return string(b)
		}
		f := make([]interface{}, 0, len(v))
		for _, av := range v {
			av = flatten(av)
			av = flatten(av, true)
			if a, ok := av.([]interface{}); ok {
				f = append(f, a...)
			} else {


@@ 643,6 668,19 @@ func flatten(v interface{}) interface{} {
	case map[string]interface{}:
		b, _ := json.Marshal(v)
		return string(b)
	case *big.Int:
		if v.IsInt64() {
			return v.Int64()
		} else {
			return v.String()
		}
	case float64:
		i := int64(v)
		f := float64(i)
		if v == f {
			return i
		}
		return v
	default:
		return v
	}


@@ 755,7 793,42 @@ func toConcrete(v string) (interface{}, error) {
		return strconv.ParseFloat(str, 64)
	case "bool", "boolean", "b":
		return strconv.ParseBool(str)
	case "jq":
		query, err := gojq.Parse(str)
		if err != nil {
			return nil, fmt.Errorf("failed to parse jq query: %w", err)
		}
		return gojq.Compile(query, gojq.WithVariables([]string{"$last"}))
	}

	return v, nil // Do not parse -- there is no associated type here.
}

func expandArgs(ctx context.Context, args []interface{}, input, last interface{}, depth int) ([]interface{}, error) {
	final := make([]interface{}, 0, len(args))
	for i, arg := range args {
		switch arg := arg.(type) {
		case []interface{}:
			arg, err := expandArgs(ctx, arg, input, last, depth+1)
			if err != nil {
				return nil, fmt.Errorf("error expanding array argument %d: %w", i, err)
			}
			final = append(final, arg)
		case *gojq.Code:
			iter := arg.RunWithContext(ctx, input, last)
			for {
				v, ok := iter.Next()
				if !ok {
					break
				}
				if err, ok := v.(error); ok {
					return nil, fmt.Errorf("error expanding jq argument: %w", err)
				}
				final = append(final, v)
			}
		default:
			final = append(final, arg)
		}
	}
	return final, nil
}

M sql.1.scd => sql.1.scd +51 -18
@@ 94,7 94,7 @@ heavily on sqlx and Go's database and JSON packages to provide much of this.
*-h*, *--help*
	Print this usage text.

## File Substitution
# File Substitution

Almost every option's value and other argument to sql(1) may be substituted by
a file by passing *@file* for its value, where file is either a path to a file


@@ 104,7 104,7 @@ the command line.
It is not recommended to use standard input more than once, as the order of
reads may not be what you expect, depending on parameter expansion.

## Output
# Output

All sql(1) output to standard output is in a JSON format. All output to
standard error is text, either for error messages or log messages about


@@ 115,7 115,7 @@ indented and broken up across multiple lines. This is the default behavior. To
compact the output, pass the *--compact* flag, which will write on JSON object
or array (if *--array* is set) per line.

## DSN URLs
# DSN URLs

The first non-optional argument to sql(1) is a DSN URL with the
following form:


@@ 139,7 139,7 @@ Currently, these are:
  `:memory:`, in which case the opened database is in-memory only --
  this has limited use, but may help with testing.

## Query
# Query

All subsequent arguments to sql(1) are SQL queries / statements and
arguments to them.


@@ 208,7 208,7 @@ $ sql ${dsn} 'select ?' str::1 ,, float::2 ,, int::3
Note in the above output that each row is for *\_\_result_0* (the default naming
for unnamed columns), instead of each being for a different columns.

## Types
# Types

Keep in mind that types are applied after file loading. So, it is possible to
give a file-based parameter a type by preceding it with that type. For example,


@@ 222,69 222,102 @@ Type: name[, aliases]
example(s)
```

Strings: *str*, *s* (default)
## Strings: *str*, *s* (default)

```
str::Foobar
```

Bytes: *bytes*, *bs*
## Bytes: *bytes*, *bs*

```
bytes::Foobar
```

Signed int (64-bit): *int*, *i*, *l*
## Signed int (64-bit): *int*, *i*, *l*

```
int::123 int::-456
```

Unsigned int (64-bit): *uint*, *u*, *ul*
## Unsigned int (64-bit): *uint*, *u*, *ul*

```
uint::123456
```

JSON array: *array*, *a*
## jq expression: *jq*

```
jq::'$last[][].id'
```

The input to any jq query is an array of all prior results, with the following
levels: query, executions, rows. So, given an sql run like the following:

```
$ sql ${dsn} 'select id from things where name = ?' foo ,, bar , 'select top from pools'
```

The jq expression `.[0][1][2]` accesses the first query's result set (`select id
from things`), its second execution (`where name = ?` with `bar`), and its third
row.

A variable for the last query executed is available as `$last`. If this is the
first query, `$last` is null.

## JSON array: *array*, *a*

```
array::'[1,2,3,4]'
```

JSON values: *json*, *j*
## JSON values: *json*, *j*

```
json::'1 2 3 4'
```

Fields (quoted, strings): *sh*
## Fields (quoted, strings): *sh*

```
sh::'"foo" "bar"'
```

Fields (quoted, nested typing): *fields*, *fs*
## Fields (quoted, nested typing): *fields*, *fs*

```
fields::'int::1 str::2'
```

File contents as a string: *openfile*, *of*
## File contents as a string: *openfile*, *of*

```
openfile::./data
(similar to @./data)
```

File contents as bytes: *rawfile*, *rf*
## File contents as bytes: *rawfile*, *rf*

```
rawfile::./data
(similar to bytes::@./data)
```

Contents of a file descriptor: *fd*
## Contents of a file descriptor: *fd*

```
fd::4
```

Floats (64-bit): *float*, *double*, *single*, *real*, *d*, *f*
## Floats (64-bit): *float*, *double*, *single*, *real*, *d*, *f*

```
float::1.23456
```

Booleans: *bool*, *boolean*, *b*
## Booleans: *bool*, *boolean*, *b*

```
bool::true, bool::0, bool::FALSE (any form supported by Go's
strconv.ParseBool)