~ach/hermes

404f004b69035a0119f094be6fbaf65d68c8829f — Andrew Chambers 3 years ago 9ed4057 wip
WIP package in package.
M src/cmd/coolpkg/build.go => src/cmd/coolpkg/build.go +1 -1
@@ 104,7 104,7 @@ func buildMain() {
		die("Got an error evaluating expression: %s\n", err)
	}

	pkg, err := buildEnv.NormalizePackage(v)
	pkg, err := pkgs.NormalizePackage(&pkgs.PackageSpace{StorePath: store.StorePath()}, v)
	if err != nil {
		die("Error compiling package: %s\n", err)
	}

M src/cmd/coolpkg/main.go => src/cmd/coolpkg/main.go +1 -1
@@ 53,7 53,7 @@ func main() {
		mainFunc = initStoreMain
	case "scan-references":
		mainFunc = scanReferencesMain
	case "fetchurl-server":
	case "fetchurl-serve":
		mainFunc = fetchurlServerMain
	default:
		badUsage(fmt.Sprintf("Unknown command '%s'", os.Args[1]))

M src/pkgs/builtins.go => src/pkgs/builtins.go +17 -0
@@ 12,6 12,7 @@ import (
	"go.starlark.net/starlark"
)

var Out *starlark.Dict // XXX this should be a special starlark value type.
var BuiltinEnv starlark.StringDict

func init() {


@@ 19,6 20,13 @@ func init() {
	//  we could lazily init.
	BuiltinEnv = make(starlark.StringDict)
	BuiltinEnv["local_file"] = starlark.NewBuiltin("local_file", localFileBuiltin)
	// XXX We need a name for this, my initial thoughts are that
	// load should be 'import' and import should be load.
	// BuiltinEnv["load2"] = starlark.NewBuiltin("load2", load2Builtin)

	// We can reference by identity.
	Out = starlark.NewDict(0)
	BuiltinEnv["Out"] = Out

	builtins := starlark.NewDict(2)
	builtins.SetKey(starlark.String("name"), starlark.String("builtins"))


@@ 92,3 100,12 @@ func localFileBuiltin(thread *starlark.Thread, fn *starlark.Builtin, args starla

	return pkg, nil
}

/*

func load2Builtin(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	currentDir := thread.Local("buildenv").(string)

}

*/

M src/pkgs/interp.go => src/pkgs/interp.go +41 -23
@@ 5,6 5,7 @@ import (
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"go.starlark.net/starlark"
)



@@ 34,52 35,69 @@ func addBuiltins(e starlark.StringDict) starlark.StringDict {
	return e
}

func newStarlarkThread(mc *ModuleCache, module string) *starlark.Thread {
	thread := &starlark.Thread{Name: "coolpkg " + module, Load: mc.Load}
	thread.SetLocal("current_dir", filepath.Dir(module))
// Given a working dir and a starlark value, returns an absolute path
type PathResolver func(workingDir string, path starlark.Value) (string, error)

func newStarlarkThread(mc *ModuleCache, dir string, resolvePath PathResolver) *starlark.Thread {
	thread := &starlark.Thread{Load: mc.Load}
	thread.SetLocal("module_cache", mc)
	thread.SetLocal("current_dir", dir)
	thread.SetLocal("resolve_path", resolvePath)
	return thread
}

func (mc *ModuleCache) Load(thread *starlark.Thread, module string) (starlark.StringDict, error) {
	if !filepath.IsAbs(module) {
		module = filepath.Join(thread.Local("current_dir").(string), module)
func (mc *ModuleCache) Load(thread *starlark.Thread, loadPath string) (starlark.StringDict, error) {
	currentDir := thread.Local("current_dir").(string)
	resolvePath := thread.Local("resolve_path").(PathResolver)

	resolvedPath, err := resolvePath(currentDir, starlark.String(loadPath))
	if err != nil {
		return nil, errors.Wrap(err, "unable to resolve load path")
	}

	e, ok := mc.lut[module]
	e, ok := mc.lut[resolvedPath]
	if e == nil {
		if ok {
			return nil, fmt.Errorf("coolpkg reference cycle")
			return nil, fmt.Errorf("circular reference detected in package loads")
		}

		mc.lut[module] = nil
		thread := newStarlarkThread(mc, module)
		globals, err := starlark.ExecFile(thread, module, nil, addBuiltins(nil))
		mc.lut[resolvedPath] = nil
		thread := newStarlarkThread(mc, filepath.Dir(resolvedPath), resolvePath)
		globals, err := starlark.ExecFile(thread, resolvedPath, nil, addBuiltins(nil))
		e = &lutEnt{globals, err}
		mc.lut[module] = e
		mc.lut[resolvedPath] = e
	}
	return e.globals, e.err
}

// LoadUrl run a coolpkg module url with starlark providing coolpkg specific
// ExecFile run a coolpkg module url with starlark providing coolpkg specific
// builtins and setup.
func LoadUrl(module string) (starlark.StringDict, error) {
	// TODO define what types of url's we accept.
	// git, other coolpkg files, https, etc.
func ExecFile(fpath string, resolvePath PathResolver) (starlark.StringDict, error) {
	mc := NewCoolpkgModuleCache()
	thread := newStarlarkThread(mc, module)
	globals, err := starlark.ExecFile(thread, module, nil, addBuiltins(nil))
	currentDir, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	resolvedPath, err := resolvePath(currentDir, starlark.String(fpath))
	if err != nil {
		return nil, err
	}

	thread := newStarlarkThread(mc, filepath.Dir(resolvedPath), resolvePath)
	globals, err := starlark.ExecFile(thread, resolvedPath, nil, addBuiltins(nil))
	if err != nil {
		return nil, err
	}
	return globals, nil
}

func Eval(expr string, env starlark.StringDict) (starlark.Value, error) {
	t := &starlark.Thread{Name: "coolpkg eval"}
	wd, err := os.Getwd()
func Eval(env starlark.StringDict, resolvePath PathResolver, expr string) (starlark.Value, error) {
	mc := NewCoolpkgModuleCache()
	currentDir, err := os.Getwd()
	if err != nil {
		return nil, err
	}
	t.SetLocal("current_dir", wd)
	return starlark.Eval(t, "eval", expr, addBuiltins(env))
	thread := newStarlarkThread(mc, currentDir, resolvePath)
	return starlark.Eval(thread, "eval", expr, addBuiltins(env))
}

A src/pkgs/pkg.go => src/pkgs/pkg.go +304 -0
@@ 0,0 1,304 @@
package pkgs

import (
	"crypto/sha256"
	"encoding/base32"
	"encoding/binary"
	"fmt"
	"hash"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"go.starlark.net/starlark"
)

type PackageSpace struct {
	StorePath string
}

// PkgPath returns the full path of a package.
func PkgPath(storepath, pkgName, pkgHash string) string {
	if pkgName == "" {
		return filepath.Join(storepath, filepath.Join("pkgs", pkgHash))
	}
	return filepath.Join(storepath, filepath.Join("pkgs", fmt.Sprintf("%s-%s", pkgHash, pkgName)))
}

type NormalizedPackage struct {
	Name        string
	Builder     []interface{} // Array of strings, normalized packages, and the placeholder 'Out'.
	Hash        string
	Content     string
	FullPkgPath string
}

func (pkg *NormalizedPackage) BuilderScript() string {
	var sb strings.Builder

	for _, b := range pkg.Builder {
		switch b := b.(type) {
		case string:
			sb.WriteString(b)
		case *NormalizedPackage:
			sb.WriteString(b.FullPkgPath)
		default:
			panic("bug")
		}
	}

	return sb.String()
}

type packageCache struct {
	pkgdCache map[*starlark.Dict]*NormalizedPackage
	pkglCache map[*starlark.List]*NormalizedPackage
}

func (cache *packageCache) Get(pkgv starlark.Value) (*NormalizedPackage, bool) {
	// XXX is this type switching needed? map of interface works?
	// XXX We could also cache by structural identity, we should only be
	// XXX dealing with frozen values, so we could use HASH and deep equal?
	// XXX This may be useful for when we have multiple instances of the same
	// XXX package...
	switch pkgv := pkgv.(type) {
	case *starlark.Dict:
		pkg, ok := cache.pkgdCache[pkgv]
		return pkg, ok
	case *starlark.List:
		pkg, ok := cache.pkglCache[pkgv]
		return pkg, ok
	default:
		return nil, false
	}
}

func (cache *packageCache) Put(pkgv starlark.Value, pkg *NormalizedPackage) {
	switch pkgv := pkgv.(type) {
	case *starlark.Dict:
		cache.pkgdCache[pkgv] = pkg
	case *starlark.List:
		cache.pkglCache[pkgv] = pkg
	}
}

func NormalizePackage(space *PackageSpace, pkgv starlark.Value) (*NormalizedPackage, error) {
	return normalizePackage(space, &packageCache{
		pkgdCache: make(map[*starlark.Dict]*NormalizedPackage),
		pkglCache: make(map[*starlark.List]*NormalizedPackage),
	}, pkgv)
}

func normalizePackage(space *PackageSpace, cache *packageCache, pkgv starlark.Value) (*NormalizedPackage, error) {
	pkg, ok := cache.Get(pkgv)
	if ok {
		if pkg == nil {
			// XXX we need a better diagnostic, this could be annoying to debug...
			return nil, errors.New("package dependency loop")
		}
		return pkg, nil
	}
	cache.Put(pkgv, nil)

	var err error

	switch pkgv := pkgv.(type) {
	case *starlark.Dict:
		pkg, err = normalizeDictPackage(space, cache, pkgv)
	case *starlark.List:
		pkg, err = normalizeListPackage(space, cache, pkgv)
	default:
		return nil, errors.Errorf("cannot build package from type %T", pkgv)
	}
	if err != nil {
		return nil, err
	}

	pkg.Hash, err = packageHash(space, pkg)
	if err != nil {
		return nil, err
	}
	pkg.FullPkgPath = PkgPath(space.StorePath, pkg.Name, pkg.Hash)
	cache.Put(pkgv, pkg)

	return pkg, nil
}

func normalizeDictPackage(space *PackageSpace, cache *packageCache, d *starlark.Dict) (*NormalizedPackage, error) {
	pkg := &NormalizedPackage{}

	// XXX error on unused members

	v, ok, err := d.Get(starlark.String("name"))
	if err != nil {
		return nil, errors.Wrap(err, "unable to fetch name value")
	}
	if ok {
		namev, ok := v.(starlark.String)
		if !ok {
			return nil, errors.Errorf("'name' field must be convertable to a string (got '%T')", v)
		}

		name := string(namev)
		if strings.Contains(name, "/") {
			return nil, errors.Errorf("'name' field of package ('%s') cannot contain '/'", name)
		}

		pkg.Name = name
	}

	v, ok, err = d.Get(starlark.String("content"))
	if err != nil {
		return nil, errors.Wrap(err, "unable to fetch content value")
	}
	if ok {
		contentv, ok := v.(starlark.String)
		if !ok {
			return nil, errors.Errorf("'content' field must be convertable to a string (got '%T')", v)
		}

		content := string(contentv)
		pkg.Content = content
	}

	builderv, ok, err := d.Get(starlark.String("builder"))
	if err != nil {
		return nil, errors.Wrap(err, "unable to get package builder value")
	}
	if !ok {
		return nil, errors.Wrap(err, "package requires a builder field")
	}

	builder, err := normalizePackageBuilder(space, cache, builderv)
	if err != nil {
		return nil, err
	}
	pkg.Builder = builder

	return pkg, nil
}

func normalizeListPackage(space *PackageSpace, cache *packageCache, l *starlark.List) (*NormalizedPackage, error) {
	pkg := &NormalizedPackage{}

	builder, err := normalizePackageBuilder(space, cache, l)
	if err != nil {
		return nil, err
	}
	pkg.Builder = builder

	return pkg, nil
}

func normalizePackageBuilder(space *PackageSpace, cache *packageCache, builderv starlark.Value) ([]interface{}, error) {
	builder := []interface{}{}

	builders, ok := builderv.(starlark.String)
	if ok {
		builder = append(builder, string(builders))
		return builder, nil
	}

	builderl, ok := builderv.(*starlark.List)
	if !ok {
		return nil, errors.New("package builders must be a or list string")
	}

	it := builderl.Iterate()
	defer it.Done()
	var subv starlark.Value
	for it.Next(&subv) {
		switch subv := subv.(type) {
		case starlark.String:
			builder = append(builder, string(subv))
		default:
			pkg, err := normalizePackage(space, cache, subv)
			if err != nil {
				return nil, err
			}
			builder = append(builder, pkg)
		}
	}

	return builder, nil
}

// XXX We should maybe use a different base?
var base32Encoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz012345").WithPadding(base32.NoPadding)

// PackageHash returns a deterministic hash for a given
// package, this hash is used to determine package identity
// and install path and is based on all members of the package.
func packageHash(space *PackageSpace, pkg *NormalizedPackage) (string, error) {
	h := sha256.New()
	// Namespace hashes to store path, new store path is a new package.
	_, err := h.Write([]byte(space.StorePath))
	if err != nil {
		return "", err
	}
	err = packageHash2(h, pkg)
	if err != nil {
		return "", err
	}
	return base32Encoding.EncodeToString(h.Sum(nil)), nil
}

func packageHash2(h hash.Hash, pkg *NormalizedPackage) error {
	err := hashString(h, pkg.Name)
	if err != nil {
		return err
	}

	err = hashString(h, pkg.Content)
	if err != nil {
		return err
	}

	err = hashInt32(h, int32(len(pkg.Builder)))
	if err != nil {
		return err
	}

	for _, v := range pkg.Builder {
		switch v := v.(type) {
		case string:
			_, err = h.Write([]byte("s"))
			if err != nil {
				return err
			}
			err = hashString(h, v)
			if err != nil {
				return err
			}
		case *NormalizedPackage:
			if v.Hash == "" {
				panic(fmt.Sprintf("%#v", v))
			}
			_, err = h.Write([]byte("p"))
			if err != nil {
				return err
			}
			_, err = h.Write([]byte(v.Hash))
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func hashInt32(h hash.Hash, n int32) error {
	var buf [4]byte
	binary.BigEndian.PutUint32(buf[:], uint32(n))
	_, err := h.Write(buf[:])
	return err
}

func hashString(h hash.Hash, s string) error {
	err := hashInt32(h, int32(len(s)))
	if err != nil {
		return err
	}
	_, err = h.Write([]byte(s))
	return err
}

M src/store/buildenv.go => src/store/buildenv.go +4 -197
@@ 8,23 8,19 @@ import (
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/andrewchambers/coolpkg/pkgs"
	"github.com/andrewchambers/coolpkg/proctools"
	"github.com/pkg/errors"
	"go.starlark.net/starlark"
)

type BuildFunc func(ctx context.Context, buildEnv *BuildEnv, pkg *NormalizedPackage) error
type BuildFunc func(ctx context.Context, buildEnv *BuildEnv, pkg *pkgs.NormalizedPackage) error

type BuildEnv struct {
	KeepEnvVars     []string
	BuildOutputSink io.Writer
	BuildFunc       BuildFunc
	storePath       string
	pkgsCache       map[starlark.String]*NormalizedPackage
	pkgdCache       map[*starlark.Dict]*NormalizedPackage
	pkglCache       map[*starlark.List]*NormalizedPackage
}

func NewBuildEnv(storePath string) (*BuildEnv, error) {


@@ 34,200 30,11 @@ func NewBuildEnv(storePath string) (*BuildEnv, error) {
	}
	buildEnv := &BuildEnv{
		storePath: absStore,
		pkgsCache: make(map[starlark.String]*NormalizedPackage),
		pkgdCache: make(map[*starlark.Dict]*NormalizedPackage),
		pkglCache: make(map[*starlark.List]*NormalizedPackage),
		BuildFunc: NSJailBuild,
	}
	return buildEnv, nil
}

func (buildEnv *BuildEnv) NormalizePackage(pkgv starlark.Value) (*NormalizedPackage, error) {
	pkg, ok := buildEnv.normalizedCacheGet(pkgv)
	if ok {
		if pkg == nil {
			return nil, errors.New("package dependency loop")
		}
		return pkg, nil
	}
	buildEnv.normalizedCachePut(pkgv, nil)

	var err error

	switch pkgv := pkgv.(type) {
	case starlark.String:
		pkg, err = buildEnv.normalizeStringPackage(pkgv)
	case *starlark.Dict:
		pkg, err = buildEnv.normalizeDictPackage(pkgv)
	case *starlark.List:
		pkg, err = buildEnv.normalizeListPackage(pkgv)
	default:
		return nil, errors.Errorf("cannot build package from type %T", pkgv)
	}
	if err != nil {
		buildEnv.normalizedCacheDel(pkgv)
		return nil, err
	}

	pkg.Hash, err = packageHash(buildEnv.storePath, pkg)
	if err != nil {
		buildEnv.normalizedCacheDel(pkgv)
		return nil, err
	}
	pkg.FullPkgPath = PkgPath(buildEnv.storePath, pkg.Name, pkg.Hash)
	buildEnv.normalizedCachePut(pkgv, pkg)

	return pkg, err
}

func (buildEnv *BuildEnv) normalizedCacheGet(pkgv starlark.Value) (*NormalizedPackage, bool) {
	// XXX is this type switching needed? map of interface works?
	// XXX We could also cache by structural identity, we should only be
	// XXX dealing with frozen values, so we could use HASH and deep equal?
	// XXX This may be useful for when we have multiple instances of the same
	// XXX package...
	switch pkgv := pkgv.(type) {
	case starlark.String:
		pkg, ok := buildEnv.pkgsCache[pkgv]
		return pkg, ok
	case *starlark.Dict:
		pkg, ok := buildEnv.pkgdCache[pkgv]
		return pkg, ok
	case *starlark.List:
		pkg, ok := buildEnv.pkglCache[pkgv]
		return pkg, ok
	default:
		return nil, false
	}
}

func (buildEnv *BuildEnv) normalizedCachePut(pkgv starlark.Value, pkg *NormalizedPackage) {
	switch pkgv := pkgv.(type) {
	case starlark.String:
		buildEnv.pkgsCache[pkgv] = pkg
	case *starlark.Dict:
		buildEnv.pkgdCache[pkgv] = pkg
	case *starlark.List:
		buildEnv.pkglCache[pkgv] = pkg
	}
}

func (buildEnv *BuildEnv) normalizedCacheDel(pkgv starlark.Value) {
	switch pkgv := pkgv.(type) {
	case starlark.String:
		delete(buildEnv.pkgsCache, pkgv)
	case *starlark.Dict:
		delete(buildEnv.pkgdCache, pkgv)
	case *starlark.List:
		delete(buildEnv.pkglCache, pkgv)
	}
}

func (buildEnv *BuildEnv) normalizeDictPackage(d *starlark.Dict) (*NormalizedPackage, error) {
	pkg := &NormalizedPackage{}

	// XXX error on unused members

	v, ok, err := d.Get(starlark.String("name"))
	if err != nil {
		return nil, errors.Wrap(err, "unable to fetch name value")
	}
	if ok {
		namev, ok := v.(starlark.String)
		if !ok {
			return nil, errors.Errorf("'name' field must be convertable to a string (got '%T')", v)
		}

		name := string(namev)
		if strings.Contains(name, "/") {
			return nil, errors.Errorf("'name' field of package ('%s') cannot contain '/'", name)
		}

		pkg.Name = name
	}

	v, ok, err = d.Get(starlark.String("content"))
	if err != nil {
		return nil, errors.Wrap(err, "unable to fetch content value")
	}
	if ok {
		contentv, ok := v.(starlark.String)
		if !ok {
			return nil, errors.Errorf("'content' field must be convertable to a string (got '%T')", v)
		}

		content := string(contentv)
		pkg.Content = content
	}

	builderv, ok, err := d.Get(starlark.String("builder"))
	if err != nil {
		return nil, errors.Wrap(err, "unable to get package builder value")
	}
	if !ok {
		return nil, errors.Wrap(err, "package requires a builder field")
	}

	builder, err := buildEnv.normalizePackageBuilder(builderv)
	if err != nil {
		return nil, err
	}
	pkg.Builder = builder

	return pkg, nil
}

func (buildEnv *BuildEnv) normalizeListPackage(l *starlark.List) (*NormalizedPackage, error) {
	pkg := &NormalizedPackage{}

	builder, err := buildEnv.normalizePackageBuilder(l)
	if err != nil {
		return nil, err
	}
	pkg.Builder = builder

	return pkg, nil
}

func (buildEnv *BuildEnv) normalizeStringPackage(s starlark.String) (*NormalizedPackage, error) {
	pkg := &NormalizedPackage{}
	pkg.Builder = []interface{}{string(s)}
	return pkg, nil
}

func (buildEnv *BuildEnv) normalizePackageBuilder(builderv starlark.Value) ([]interface{}, error) {
	builder := []interface{}{}

	builders, ok := builderv.(starlark.String)
	if ok {
		builder = append(builder, string(builders))
		return builder, nil
	}

	builderl, ok := builderv.(*starlark.List)
	if !ok {
		return nil, errors.New("package builders must be a or list string")
	}

	it := builderl.Iterate()
	defer it.Done()
	var subv starlark.Value
	for it.Next(&subv) {
		switch subv := subv.(type) {
		case starlark.String:
			builder = append(builder, string(subv))
		default:
			pkg, err := buildEnv.NormalizePackage(subv)
			if err != nil {
				return nil, err
			}
			builder = append(builder, pkg)
		}
	}

	return builder, nil
}

// Start a coolpkg fetchurl server and return a closure which
// stops it and cleans up any resources/temporary directories
// it may have created.


@@ 308,7 115,7 @@ func startFetchurlServer(ctx context.Context, socketpath string, hashRequired bo
	return cleanup, nil
}

func UnsandboxedBuild(ctx context.Context, buildEnv *BuildEnv, pkg *NormalizedPackage) error {
func UnsandboxedBuild(ctx context.Context, buildEnv *BuildEnv, pkg *pkgs.NormalizedPackage) error {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		return err


@@ 363,7 170,7 @@ func UnsandboxedBuild(ctx context.Context, buildEnv *BuildEnv, pkg *NormalizedPa
	return nil
}

func NSJailBuild(ctx context.Context, buildEnv *BuildEnv, pkg *NormalizedPackage) error {
func NSJailBuild(ctx context.Context, buildEnv *BuildEnv, pkg *pkgs.NormalizedPackage) error {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		return err

D src/store/pkg.go => src/store/pkg.go +0 -115
@@ 1,115 0,0 @@
package store

import (
	"crypto/sha256"
	"encoding/base32"
	"encoding/binary"
	"fmt"
	"hash"
	"strings"
)

type NormalizedPackage struct {
	Name        string
	Builder     []interface{} // Array of strings and normalized packages.
	Hash        string
	FullPkgPath string
	Content     string
}

func (pkg *NormalizedPackage) BuilderScript() string {
	var sb strings.Builder

	for _, b := range pkg.Builder {
		switch b := b.(type) {
		case string:
			sb.WriteString(b)
		case *NormalizedPackage:
			sb.WriteString(b.FullPkgPath)
		default:
			panic("bug")
		}
	}

	return sb.String()
}

// XXX We should maybe use a different base?
var base32Encoding = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz012345").WithPadding(base32.NoPadding)

// PackageHash returns a deterministic hash for a given
// package, this hash is used to determine package identity
// and install path and is based on all members of the package.
func packageHash(storePath string, pkg *NormalizedPackage) (string, error) {
	h := sha256.New()
	// Namespace hashes to store path, new store path is a new package.
	_, err := h.Write([]byte(storePath))
	if err != nil {
		return "", err
	}
	err = packageHash2(h, pkg)
	if err != nil {
		return "", err
	}
	return base32Encoding.EncodeToString(h.Sum(nil)), nil
}

func packageHash2(h hash.Hash, pkg *NormalizedPackage) error {
	err := hashString(h, pkg.Name)
	if err != nil {
		return err
	}

	err = hashString(h, pkg.Content)
	if err != nil {
		return err
	}

	err = hashInt32(h, int32(len(pkg.Builder)))
	if err != nil {
		return err
	}

	for _, v := range pkg.Builder {
		switch v := v.(type) {
		case string:
			_, err = h.Write([]byte("s"))
			if err != nil {
				return err
			}
			err = hashString(h, v)
			if err != nil {
				return err
			}
		case *NormalizedPackage:
			if v.Hash == "" {
				panic(fmt.Sprintf("%#v", v))
			}
			_, err = h.Write([]byte("p"))
			if err != nil {
				return err
			}
			err = packageHash2(h, v)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func hashInt32(h hash.Hash, n int32) error {
	var buf [4]byte
	binary.BigEndian.PutUint32(buf[:], uint32(n))
	_, err := h.Write(buf[:])
	return err
}

func hashString(h hash.Hash, s string) error {
	err := hashInt32(h, int32(len(s)))
	if err != nil {
		return err
	}
	_, err = h.Write([]byte(s))
	return err
}

M src/store/store.go => src/store/store.go +8 -16
@@ 5,7 5,6 @@ import (
	"container/list"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"


@@ 16,6 15,7 @@ import (

	"github.com/andrewchambers/coolpkg/dtar"
	"github.com/andrewchambers/coolpkg/hhash"
	"github.com/andrewchambers/coolpkg/pkgs"
	"github.com/bvinc/go-sqlite-lite/sqlite3"
	"github.com/gofrs/flock"
	"github.com/pkg/errors"


@@ 195,16 195,8 @@ func (store *Store) PkgPathFromHash(hash string) (string, error) {
}

// PkgPath returns the full path of a package.
func PkgPath(storepath, pkgName, pkgHash string) string {
	if pkgName == "" {
		return filepath.Join(storepath, filepath.Join("pkgs", pkgHash))
	}
	return filepath.Join(storepath, filepath.Join("pkgs", fmt.Sprintf("%s-%s", pkgHash, pkgName)))
}

// PkgPath returns the full path of a package.
func (store *Store) PkgPath(pkgName, pkgHash string) string {
	return PkgPath(store.storePath, pkgName, pkgHash)
	return pkgs.PkgPath(store.storePath, pkgName, pkgHash)
}

func rmTree(treePath string) error {


@@ 574,14 566,14 @@ func (store *Store) CollectGarbage() error {
// "builder" - A build script containing input substitution templates of the form '${NAME}'.
// "inputs"  - A dict mapping input names to other packages.
//
func (store *Store) BuildPackage(ctx context.Context, env *BuildEnv, pkg *NormalizedPackage, outLink *string) error {
func (store *Store) BuildPackage(ctx context.Context, env *BuildEnv, pkg *pkgs.NormalizedPackage, outLink *string) error {
	return store.buildPackage(ctx, env, pkg, env.BuildFunc, outLink)
}

// InjectTarball pretends to build a package imitating BuildPackage, but actually
// takes the build result from the contents of a tarball instead of executing the package builder.
func (store *Store) InjectTarball(ctx context.Context, env *BuildEnv, pkg *NormalizedPackage, tr *tar.Reader, outLink *string) error {
	return store.buildPackage(ctx, env, pkg, func(ctx context.Context, env *BuildEnv, pkg *NormalizedPackage) error {
func (store *Store) InjectTarball(ctx context.Context, env *BuildEnv, pkg *pkgs.NormalizedPackage, tr *tar.Reader, outLink *string) error {
	return store.buildPackage(ctx, env, pkg, func(ctx context.Context, env *BuildEnv, pkg *pkgs.NormalizedPackage) error {
		err := untar(tr, pkg.FullPkgPath)
		if err != nil {
			return errors.Wrapf(err, "unable to inject tarball into package store at '%s'", pkg.FullPkgPath)


@@ 678,7 670,7 @@ func (store *Store) AddGCRoot(linkPath, pkgPath string) error {
	return nil
}

func (store *Store) buildPackage(ctx context.Context, buildEnv *BuildEnv, pkg *NormalizedPackage, f BuildFunc, outLink *string) error {
func (store *Store) buildPackage(ctx context.Context, buildEnv *BuildEnv, pkg *pkgs.NormalizedPackage, f BuildFunc, outLink *string) error {
	if !store.gcLock.RLocked() {

		if store.buildLock.Locked() {


@@ 728,9 720,9 @@ func (store *Store) buildPackage(ctx context.Context, buildEnv *BuildEnv, pkg *N
		return errors.Wrapf(err, "unable to create '%s' in package database", pkg.FullPkgPath)
	}

	alreadyBuilt := make(map[*NormalizedPackage]struct{})
	alreadyBuilt := make(map[*pkgs.NormalizedPackage]struct{})
	for _, b := range pkg.Builder {
		dep, isPkg := b.(*NormalizedPackage)
		dep, isPkg := b.(*pkgs.NormalizedPackage)
		if isPkg {
			_, isBuilt := alreadyBuilt[dep]
			if !isBuilt {

M support/shell.nix => support/shell.nix +5 -1
@@ 1,5 1,9 @@
let 
  pkgs = (import <nixpkgs> {});
  pkgs = (import (builtins.fetchTarball {
    name = "nixpkgs";
    url = https://releases.nixos.org/nixpkgs/nixpkgs-19.09pre188239.c0e56afddbc/nixexprs.tar.xz;
    sha256 = "02sijmad7jybzwf063aig6bsaw4h85as0ax4x2425c867n62xnxz";
  }) {});
in
  pkgs.mkShell {
      buildInputs = with pkgs; [