12 files changed, 286 insertions(+), 25 deletions(-)

M TODO.txt
M csrc/hermes-linux-namespace-sandbox.c
M do
A src/cmd/builtins/copy.go
M src/cmd/builtins/main.go
A src/cmd/builtins/mkdir.go
M src/cmd/builtins/unpack.go
M src/cmd/hermes/fetchserver.go
M src/pkgs/interp/builtins.go
M src/pkgs/pkg.go
M src/store/init.go
A tests/devtests/0012-pkgtree.test

@@ 58,6 58,7 @@ It seems useful to see it regardless. This is a maybe as it is bad for performan
   Possibly via a builtin command that can be run. @{builtins}/debug
 - Interactive debug should perhaps be automatically entered on failure?.
 - Make C++ runtime it's own package somehow?
+- Error messages for bad loads.
 
 Maybe:
 

@@ 383,6 383,15 @@ static int post_clone(void *argv) {
         NULL,
     };
 
+    if (verbose) {
+      fprintf(stderr, "execve: [");
+      char **arg = builder;
+      while (*arg) {
+        fprintf(stderr, " %s", *arg);
+        arg += 1;
+      }
+      fprintf(stderr, " ]\n");
+    }
     execve(builder[0], builder, env);
     die("execve: %s", strerror(errno));
     return 1;

@@ 22,9 22,9 @@ vcmd () {
 build_go_bin () {
   unset GOPATH
   out="$(pwd)/$2/$1"
-  pushd "src/cmd/$1"
-  vcmd go build $HERMES_GO_EXTRA_BUILD_FLAGS -mod=vendor -tags "netgo" -ldflags "-extldflags -static" -o "$out"
-  popd
+  pushd "src/cmd/$1" > /dev/null
+  vcmd go build $HERMES_GO_EXTRA_BUILD_FLAGS -mod=vendor -ldflags "-linkmode=external -extldflags -static" -o "$out"
+  popd > /dev/null
 }
 
 test -f "config.inc" || fail "please run ./configure first."


@@ 44,7 44,7 @@ case "$1" in
     
     build_go_bin builtins build/builtins
 
-    for cmd in fetch unpack writefile runcmds
+    for cmd in fetch unpack writefile runcmds mkdir copy
     do
        (cd "build/builtins" && ln -s ./builtins "$cmd")
     done


@@ 59,9 59,7 @@ EOF
     
     build_go_bin hermes build/sysroot/bin
     build_go_bin hermes-pkgstore build/sysroot/libexec/hermes
-
     ${CC} ${CFLAGS} ${LDFLAGS} -o build/sysroot/libexec/hermes/hermes-linux-namespace-sandbox csrc/hermes-linux-namespace-sandbox.c
-
   ;;
   
   install)

@@ 0,0 1,128 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	flag "github.com/spf13/pflag"
+)
+
+var skipReList []*regexp.Regexp
+
+func copy(src, dest string) error {
+	info, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+	return copy2(src, dest, info)
+}
+
+func copy2(src, dest string, info os.FileInfo) error {
+
+	for _, skipRe := range skipReList {
+		if skipRe.MatchString(src) {
+			return nil
+		}
+	}
+
+	if info.Mode()&os.ModeSymlink != 0 {
+
+		src, err := os.Readlink(src)
+		if err != nil {
+			return err
+		}
+
+		return os.Symlink(src, dest)
+	} else if info.IsDir() {
+
+		originalMode := info.Mode()
+
+		err := os.MkdirAll(dest, os.FileMode(0755))
+		if err != nil {
+			return err
+		}
+
+		contents, err := ioutil.ReadDir(src)
+		if err != nil {
+			return err
+		}
+
+		for _, content := range contents {
+			cs, cd := filepath.Join(src, content.Name()), filepath.Join(dest, content.Name())
+			err := copy2(cs, cd, content)
+			if err != nil {
+				return err
+			}
+		}
+
+		err = os.Chmod(dest, originalMode)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	} else if info.Mode().IsRegular() {
+
+		err := os.MkdirAll(filepath.Dir(dest), os.ModePerm)
+		if err != nil {
+			return err
+		}
+
+		f, err := os.Create(dest)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
+		err = os.Chmod(f.Name(), info.Mode())
+		if err != nil {
+			return err
+		}
+
+		s, err := os.Open(src)
+		if err != nil {
+			return err
+		}
+		defer s.Close()
+
+		_, err = io.Copy(f, s)
+		if err != nil {
+			return err
+		}
+
+		err = f.Close()
+		if err != nil {
+			return err
+		}
+
+		return nil
+	} else {
+		return fmt.Errorf("don't know how to copy path %q", src)
+	}
+}
+
+func copyMain() {
+	skip := flag.StringArray("skip", []string{}, "A list of regular expressions matched against files to skip.")
+	flag.Parse()
+
+	for _, s := range *skip {
+		r, err := regexp.Compile(s)
+		if err != nil {
+			die("Error compiling regex %q: %s\n", s, err)
+		}
+		skipReList = append(skipReList, r)
+	}
+
+	if len(flag.Args()) != 2 {
+		die("Expected SRC and DEST.\n")
+	}
+
+	src := flag.Args()[0]
+	dst := flag.Args()[1]
+
+	check(copy(src, dst))
+}

@@ 36,7 36,11 @@ func main() {
 		runcmdsMain()
 	case "writefile":
 		writefileMain()
+	case "mkdir":
+		mkdirMain()
+	case "copy":
+		copyMain()
 	default:
-		die("Please run with argv[0] as as one of \"builtins\", \"fetch\", \"runcmds\", \"writefile\" ,or \"unpack\", got %q.\n", prog)
+		die("Please run with argv[0] as as one of \"builtins\", \"fetch\", \"mkdir\", \"copy\", \"runcmds\", \"writefile\", or \"unpack\", got %q.\n", prog)
 	}
 }

@@ 0,0 1,26 @@
+package main
+
+import (
+	"os"
+
+	flag "github.com/spf13/pflag"
+)
+
+func mkdirMain() {
+	parents := flag.BoolP("parents", "p", false, "no error if existing, make parent directories as needed.")
+
+	flag.Parse()
+
+	if len(flag.Args()) != 1 {
+		die("Expected a single argument.\n")
+	}
+
+	mode := os.FileMode(0755)
+
+	dir := flag.Args()[0]
+	if *parents {
+		check(os.MkdirAll(dir, mode))
+	} else {
+		check(os.Mkdir(dir, mode))
+	}
+}

@@ 21,6 21,7 @@ func untar(r io.Reader, dest string, noUnwrap bool) {
 
 	dir, err := filepath.Abs(dest)
 	check(err)
+	check(os.MkdirAll(dir, 0755))
 
 	topLevelCount := 0
 	lastTopLevelItem := ""

@@ 4,9 4,11 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"os/signal"
 
 	"github.com/andrewchambers/hermes/fetch"
 	flag "github.com/spf13/pflag"
+	"golang.org/x/sys/unix"
 	"olympos.io/encoding/edn"
 )
 


@@ 56,6 58,16 @@ func fetchServerMain() {
 		}
 	}
 
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, os.Interrupt, unix.SIGTERM, unix.SIGHUP)
+	go func() {
+		<-sigs
+		// XXX The fetch server doesn't have much state we need to cleanup.
+		// we should use a context anyway however.
+		_ = os.Remove(opts.Socket)
+		os.Exit(0)
+	}()
+
 	l, err := net.Listen("unix", opts.Socket)
 	if err != nil {
 		die("Error listening on socket: %s\n", err)


@@ 63,5 75,6 @@ func fetchServerMain() {
 	_, err = fmt.Println("up")
 
 	err = fetch.ServeRequests(l, mirrorDB, opts.WorkDir)
+	_ = os.Remove(opts.Socket)
 	die("Server exiting: %s\n", err)
 }

@@ 149,9 149,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
 	mirrors := thread.Local("mirror_db").(*MirrorMap)
 	currentDir := thread.Local("current_dir").(string)
 
-	hash := ""
+	fnname := fn.Name()
+	contentHash := ""
 
-	err := hscript.UnpackArgs(fn.Name(), args, kwargs, "hash", &hash)
+	importSpecs := args
+	err := hscript.UnpackArgs(fnname, hscript.Tuple{}, kwargs, "hash", &contentHash)
 	if err != nil {
 		return nil, err
 	}


@@ 163,10 165,10 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
 	})
 
 	malformedArgError := func(idx int) error {
-		return errors.Errorf("argument %d should be of the form [\"importpath\", \"url\" | package]", idx)
+		return errors.Errorf("%s: argument %d should be of the form [\"importpath\", \"url\" | package]", fnname, idx)
 	}
 
-	for argIdx, imp := range args {
+	for argIdx, imp := range importSpecs {
 		switch imp := imp.(type) {
 		case *hscript.List:
 			if imp.Len() != 2 {


@@ 188,21 190,38 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
 				// we are importing from a url string.
 				fetchUrl := string(importContent)
 				file_name := ""
+				fetchHash := ""
 				unpack := true
 				unpack_unwrap := true
 				unpack_type := ""
 
-				err = processAndValidateFetchUrl(fn, mirrors, currentDir, &hash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
+				err = processAndValidateFetchUrl(fn, mirrors, currentDir, &fetchHash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
 				if err != nil {
 					return nil, err
 				}
 
+				urlArg := ""
+				if fetchUrl != "" {
+					urlArg = " " + strconv.Quote("--url="+fetchUrl)
+				}
+
+				hashArg := ""
+				if fetchHash != "" {
+					hashArg = " " + strconv.Quote("--hash="+fetchHash)
+				}
+
+				unpackTypeArg := ""
+				if unpack_type != "" {
+					unpackTypeArg = " " + strconv.Quote("--type="+unpack_type)
+				}
+
 				builder.Append(Placeholder{"builtins"})
 				builder.Append(
 					hscript.String(
 						fmt.Sprintf(
-							"/fetch %q %q\n",
-							"--url="+fetchUrl,
+							"/fetch%s%s %q\n",
+							urlArg,
+							hashArg,
 							"--out="+file_name,
 						),
 					),


@@ 211,7 230,8 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
 				builder.Append(
 					hscript.String(
 						fmt.Sprintf(
-							"/unpack %q %q\n",
+							"/unpack%s %q %q\n",
+							unpackTypeArg,
 							"--dest=${out}/"+importPath,
 							file_name,
 						),


@@ 221,7 241,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
 			case *hscript.List, *hscript.Dict:
 				// We are importing from a package.
 				builder.Append(Placeholder{"builtins"})
-				builder.Append(hscript.String("/symlink \""))
+				builder.Append(hscript.String(fmt.Sprintf("/mkdir -p %q\n", "${out}/"+filepath.Dir(importPath))))
+				builder.Append(Placeholder{"builtins"})
+				// We choose to copy instead of link, so the package content hash does not depend on the package path.
+				// when copying we also skip the .hermes_pkg file as that contains the store path.
+				builder.Append(hscript.String("/copy --skip \"[^\\.]+\\.hermes_pkg$\" \""))
 				builder.Append(importContent)
 				builder.Append(hscript.String(fmt.Sprintf("\" %q\n", "${out}/"+importPath)))
 			default:


@@ 231,8 255,9 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
 	}
 
 	d := &hscript.Dict{}
+	d.SetKey(hscript.String("name"), hscript.String("pkg-tree"))
 	d.SetKey(hscript.String("builder"), builder)
-	d.SetKey(hscript.String("content"), hscript.String(hash))
+	d.SetKey(hscript.String("content"), hscript.String(contentHash))
 	return d, nil
 }
 

@@ 92,15 92,32 @@ func DecodeAddressFromPath(storePath, pkgPath string) (string, error) {
 	return pkgHashAndName, nil
 }
 
-func ValidPackagePathChar(r rune) error {
+// XXX whitelist better?
+func ValidStorePathChar(r rune) error {
 	switch r {
 	case '\r', '\n':
-		return errors.New("package paths must not contain new lines")
+		return errors.New("store paths must not contain new lines")
 	case '"', '\'':
-		return errors.New("package paths must not contain quotations")
-	case '/', '\\':
-		return errors.New("package paths must not contain '\\' or '/'")
-		// XXX It may be useful to disallow whitespace so package expressions.
+		return errors.New("store paths must not contain quotations")
+	case '\\', '$':
+		return errors.Errorf("store paths must not contain %v", r)
+	case ' ', '\t':
+		return errors.New("store paths must not contain whitespace")
+	}
+	return nil
+}
+
+// XXX whitelist better?
+func ValidPackageNameChar(r rune) error {
+	switch r {
+	case '\r', '\n':
+		return errors.New("package names must not contain new lines")
+	case '"', '\'':
+		return errors.New("package names must not contain quotations")
+	case '/', '\\', '$':
+		return errors.Errorf("package names must not contain %v", r)
+	case ' ', '\t':
+		return errors.New("package names must not contain whitespace")
 	}
 	return nil
 }


@@ 111,7 128,7 @@ func ValidName(name string) error {
 		return errors.Errorf("package name '%s' is longer than the maximum name length (%d): ", name, MAX_NAME_LEN)
 	}
 	for _, r := range name {
-		err := ValidPackagePathChar(r)
+		err := ValidPackageNameChar(r)
 		if err != nil {
 			return err
 		}


@@ 121,7 138,7 @@ func ValidName(name string) error {
 
 func ValidAddress(address string) error {
 	for _, r := range address {
-		err := ValidPackagePathChar(r)
+		err := ValidPackageNameChar(r)
 		if err != nil {
 			return err
 		}

@@ 7,6 7,7 @@ import (
 	"path/filepath"
 
 	"github.com/andrewchambers/hermes/extrasqlite"
+	"github.com/andrewchambers/hermes/pkgs"
 	"github.com/bvinc/go-sqlite-lite/sqlite3"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"


@@ 23,6 24,12 @@ type StoreInitConfigTemplate struct {
 // InitStore creates a hermes store and initializes the database.
 // InitStore will not overwrite a store that has already been initialized
 func InitStore(storePath string, configTemplate StoreInitConfigTemplate) error {
+	for _, r := range storePath {
+		if err := pkgs.ValidStorePathChar(r); err != nil {
+			return err
+		}
+	}
+
 	err := os.MkdirAll(storePath, 0755)
 	if err != nil {
 		return errors.Wrap(err, "error while making store dir")

@@ 0,0 1,32 @@
+#! /usr/bin/env bash
+
+set -eu
+cd "$( dirname "${BASH_SOURCE[0]}" )" 
+
+. ./devtests.inc
+
+mkdir a
+echo "a = fetch(\"./a.txt\")"  > ./a/a.hpkg
+echo a > ./a/a.txt
+mkdir b
+echo "b = fetch(\"./b.txt\")"  > ./b/b.hpkg
+echo b > ./b/b.txt
+
+cat << 'EOF' > ./t.hpkg
+hpkgs = pkg_tree(
+  ["a", fetch("./a")],
+  ["b", "./b"],
+  hash = "sha256:0839331cbf90c1df717d83658f64b0449c3de17e7c94cd0dc8c55d054c38320e"
+)
+
+load([hpkgs, "/a/a.hpkg"], "a")
+load([hpkgs, "/b/b.hpkg"], "b")
+
+p1 = a
+p2 = b
+EOF
+
+hermes build ./t.hpkg --verbose -e p1
+diff -u ./result/a.txt ./a/a.txt
+hermes build ./t.hpkg --verbose -e p2
+diff -u ./result/b.txt ./b/b.txt