~ach/hermes unlisted

4f42a9687c566f9dbed3416a64c6ce274038c949 — Andrew Chambers 1 year, 1 month ago 0230e4d
Tests and fixes for broken tests.
M TODO.txt => TODO.txt +1 -0
@@ 58,6 58,7 @@ It seems useful to see it regardless. This is a maybe as it is bad for performan
  Possibly via a builtin command that can be run. @{builtins}/debug
- Interactive debug should perhaps be automatically entered on failure?.
- Make C++ runtime it's own package somehow?
- Error messages for bad loads.

Maybe:


M csrc/hermes-linux-namespace-sandbox.c => csrc/hermes-linux-namespace-sandbox.c +9 -0
@@ 383,6 383,15 @@ static int post_clone(void *argv) {
        NULL,
    };

    if (verbose) {
      fprintf(stderr, "execve: [");
      char **arg = builder;
      while (*arg) {
        fprintf(stderr, " %s", *arg);
        arg += 1;
      }
      fprintf(stderr, " ]\n");
    }
    execve(builder[0], builder, env);
    die("execve: %s", strerror(errno));
    return 1;

M do => do +4 -6
@@ 22,9 22,9 @@ vcmd () {
build_go_bin () {
  unset GOPATH
  out="$(pwd)/$2/$1"
  pushd "src/cmd/$1"
  vcmd go build $HERMES_GO_EXTRA_BUILD_FLAGS -mod=vendor -tags "netgo" -ldflags "-extldflags -static" -o "$out"
  popd
  pushd "src/cmd/$1" > /dev/null
  vcmd go build $HERMES_GO_EXTRA_BUILD_FLAGS -mod=vendor -ldflags "-linkmode=external -extldflags -static" -o "$out"
  popd > /dev/null
}

test -f "config.inc" || fail "please run ./configure first."


@@ 44,7 44,7 @@ case "$1" in
    
    build_go_bin builtins build/builtins

    for cmd in fetch unpack writefile runcmds
    for cmd in fetch unpack writefile runcmds mkdir copy
    do
       (cd "build/builtins" && ln -s ./builtins "$cmd")
    done


@@ 59,9 59,7 @@ EOF
    
    build_go_bin hermes build/sysroot/bin
    build_go_bin hermes-pkgstore build/sysroot/libexec/hermes

    ${CC} ${CFLAGS} ${LDFLAGS} -o build/sysroot/libexec/hermes/hermes-linux-namespace-sandbox csrc/hermes-linux-namespace-sandbox.c

  ;;
  
  install)

A src/cmd/builtins/copy.go => src/cmd/builtins/copy.go +128 -0
@@ 0,0 1,128 @@
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	flag "github.com/spf13/pflag"
)

var skipReList []*regexp.Regexp

func copy(src, dest string) error {
	info, err := os.Lstat(src)
	if err != nil {
		return err
	}
	return copy2(src, dest, info)
}

func copy2(src, dest string, info os.FileInfo) error {

	for _, skipRe := range skipReList {
		if skipRe.MatchString(src) {
			return nil
		}
	}

	if info.Mode()&os.ModeSymlink != 0 {

		src, err := os.Readlink(src)
		if err != nil {
			return err
		}

		return os.Symlink(src, dest)
	} else if info.IsDir() {

		originalMode := info.Mode()

		err := os.MkdirAll(dest, os.FileMode(0755))
		if err != nil {
			return err
		}

		contents, err := ioutil.ReadDir(src)
		if err != nil {
			return err
		}

		for _, content := range contents {
			cs, cd := filepath.Join(src, content.Name()), filepath.Join(dest, content.Name())
			err := copy2(cs, cd, content)
			if err != nil {
				return err
			}
		}

		err = os.Chmod(dest, originalMode)
		if err != nil {
			return err
		}

		return nil
	} else if info.Mode().IsRegular() {

		err := os.MkdirAll(filepath.Dir(dest), os.ModePerm)
		if err != nil {
			return err
		}

		f, err := os.Create(dest)
		if err != nil {
			return err
		}
		defer f.Close()

		err = os.Chmod(f.Name(), info.Mode())
		if err != nil {
			return err
		}

		s, err := os.Open(src)
		if err != nil {
			return err
		}
		defer s.Close()

		_, err = io.Copy(f, s)
		if err != nil {
			return err
		}

		err = f.Close()
		if err != nil {
			return err
		}

		return nil
	} else {
		return fmt.Errorf("don't know how to copy path %q", src)
	}
}

func copyMain() {
	skip := flag.StringArray("skip", []string{}, "A list of regular expressions matched against files to skip.")
	flag.Parse()

	for _, s := range *skip {
		r, err := regexp.Compile(s)
		if err != nil {
			die("Error compiling regex %q: %s\n", s, err)
		}
		skipReList = append(skipReList, r)
	}

	if len(flag.Args()) != 2 {
		die("Expected SRC and DEST.\n")
	}

	src := flag.Args()[0]
	dst := flag.Args()[1]

	check(copy(src, dst))
}

M src/cmd/builtins/main.go => src/cmd/builtins/main.go +5 -1
@@ 36,7 36,11 @@ func main() {
		runcmdsMain()
	case "writefile":
		writefileMain()
	case "mkdir":
		mkdirMain()
	case "copy":
		copyMain()
	default:
		die("Please run with argv[0] as as one of \"builtins\", \"fetch\", \"runcmds\", \"writefile\" ,or \"unpack\", got %q.\n", prog)
		die("Please run with argv[0] as as one of \"builtins\", \"fetch\", \"mkdir\", \"copy\", \"runcmds\", \"writefile\", or \"unpack\", got %q.\n", prog)
	}
}

A src/cmd/builtins/mkdir.go => src/cmd/builtins/mkdir.go +26 -0
@@ 0,0 1,26 @@
package main

import (
	"os"

	flag "github.com/spf13/pflag"
)

func mkdirMain() {
	parents := flag.BoolP("parents", "p", false, "no error if existing, make parent directories as needed.")

	flag.Parse()

	if len(flag.Args()) != 1 {
		die("Expected a single argument.\n")
	}

	mode := os.FileMode(0755)

	dir := flag.Args()[0]
	if *parents {
		check(os.MkdirAll(dir, mode))
	} else {
		check(os.Mkdir(dir, mode))
	}
}

M src/cmd/builtins/unpack.go => src/cmd/builtins/unpack.go +1 -0
@@ 21,6 21,7 @@ func untar(r io.Reader, dest string, noUnwrap bool) {

	dir, err := filepath.Abs(dest)
	check(err)
	check(os.MkdirAll(dir, 0755))

	topLevelCount := 0
	lastTopLevelItem := ""

M src/cmd/hermes/fetchserver.go => src/cmd/hermes/fetchserver.go +13 -0
@@ 4,9 4,11 @@ import (
	"fmt"
	"net"
	"os"
	"os/signal"

	"github.com/andrewchambers/hermes/fetch"
	flag "github.com/spf13/pflag"
	"golang.org/x/sys/unix"
	"olympos.io/encoding/edn"
)



@@ 56,6 58,16 @@ func fetchServerMain() {
		}
	}

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt, unix.SIGTERM, unix.SIGHUP)
	go func() {
		<-sigs
		// XXX The fetch server doesn't have much state we need to cleanup.
		// we should use a context anyway however.
		_ = os.Remove(opts.Socket)
		os.Exit(0)
	}()

	l, err := net.Listen("unix", opts.Socket)
	if err != nil {
		die("Error listening on socket: %s\n", err)


@@ 63,5 75,6 @@ func fetchServerMain() {
	_, err = fmt.Println("up")

	err = fetch.ServeRequests(l, mirrorDB, opts.WorkDir)
	_ = os.Remove(opts.Socket)
	die("Server exiting: %s\n", err)
}

M src/pkgs/interp/builtins.go => src/pkgs/interp/builtins.go +35 -10
@@ 149,9 149,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
	mirrors := thread.Local("mirror_db").(*MirrorMap)
	currentDir := thread.Local("current_dir").(string)

	hash := ""
	fnname := fn.Name()
	contentHash := ""

	err := hscript.UnpackArgs(fn.Name(), args, kwargs, "hash", &hash)
	importSpecs := args
	err := hscript.UnpackArgs(fnname, hscript.Tuple{}, kwargs, "hash", &contentHash)
	if err != nil {
		return nil, err
	}


@@ 163,10 165,10 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
	})

	malformedArgError := func(idx int) error {
		return errors.Errorf("argument %d should be of the form [\"importpath\", \"url\" | package]", idx)
		return errors.Errorf("%s: argument %d should be of the form [\"importpath\", \"url\" | package]", fnname, idx)
	}

	for argIdx, imp := range args {
	for argIdx, imp := range importSpecs {
		switch imp := imp.(type) {
		case *hscript.List:
			if imp.Len() != 2 {


@@ 188,21 190,38 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
				// we are importing from a url string.
				fetchUrl := string(importContent)
				file_name := ""
				fetchHash := ""
				unpack := true
				unpack_unwrap := true
				unpack_type := ""

				err = processAndValidateFetchUrl(fn, mirrors, currentDir, &hash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
				err = processAndValidateFetchUrl(fn, mirrors, currentDir, &fetchHash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
				if err != nil {
					return nil, err
				}

				urlArg := ""
				if fetchUrl != "" {
					urlArg = " " + strconv.Quote("--url="+fetchUrl)
				}

				hashArg := ""
				if fetchHash != "" {
					hashArg = " " + strconv.Quote("--hash="+fetchHash)
				}

				unpackTypeArg := ""
				if unpack_type != "" {
					unpackTypeArg = " " + strconv.Quote("--type="+unpack_type)
				}

				builder.Append(Placeholder{"builtins"})
				builder.Append(
					hscript.String(
						fmt.Sprintf(
							"/fetch %q %q\n",
							"--url="+fetchUrl,
							"/fetch%s%s %q\n",
							urlArg,
							hashArg,
							"--out="+file_name,
						),
					),


@@ 211,7 230,8 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
				builder.Append(
					hscript.String(
						fmt.Sprintf(
							"/unpack %q %q\n",
							"/unpack%s %q %q\n",
							unpackTypeArg,
							"--dest=${out}/"+importPath,
							file_name,
						),


@@ 221,7 241,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
			case *hscript.List, *hscript.Dict:
				// We are importing from a package.
				builder.Append(Placeholder{"builtins"})
				builder.Append(hscript.String("/symlink \""))
				builder.Append(hscript.String(fmt.Sprintf("/mkdir -p %q\n", "${out}/"+filepath.Dir(importPath))))
				builder.Append(Placeholder{"builtins"})
				// We choose to copy instead of link, so the package content hash does not depend on the package path.
				// when copying we also skip the .hermes_pkg file as that contains the store path.
				builder.Append(hscript.String("/copy --skip \"[^\\.]+\\.hermes_pkg$\" \""))
				builder.Append(importContent)
				builder.Append(hscript.String(fmt.Sprintf("\" %q\n", "${out}/"+importPath)))
			default:


@@ 231,8 255,9 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
	}

	d := &hscript.Dict{}
	d.SetKey(hscript.String("name"), hscript.String("pkg-tree"))
	d.SetKey(hscript.String("builder"), builder)
	d.SetKey(hscript.String("content"), hscript.String(hash))
	d.SetKey(hscript.String("content"), hscript.String(contentHash))
	return d, nil
}


M src/pkgs/pkg.go => src/pkgs/pkg.go +25 -8
@@ 92,15 92,32 @@ func DecodeAddressFromPath(storePath, pkgPath string) (string, error) {
	return pkgHashAndName, nil
}

func ValidPackagePathChar(r rune) error {
// XXX whitelist better?
func ValidStorePathChar(r rune) error {
	switch r {
	case '\r', '\n':
		return errors.New("package paths must not contain new lines")
		return errors.New("store paths must not contain new lines")
	case '"', '\'':
		return errors.New("package paths must not contain quotations")
	case '/', '\\':
		return errors.New("package paths must not contain '\\' or '/'")
		// XXX It may be useful to disallow whitespace so package expressions.
		return errors.New("store paths must not contain quotations")
	case '\\', '$':
		return errors.Errorf("store paths must not contain %v", r)
	case ' ', '\t':
		return errors.New("store paths must not contain whitespace")
	}
	return nil
}

// XXX whitelist better?
func ValidPackageNameChar(r rune) error {
	switch r {
	case '\r', '\n':
		return errors.New("package names must not contain new lines")
	case '"', '\'':
		return errors.New("package names must not contain quotations")
	case '/', '\\', '$':
		return errors.Errorf("package names must not contain %v", r)
	case ' ', '\t':
		return errors.New("package names must not contain whitespace")
	}
	return nil
}


@@ 111,7 128,7 @@ func ValidName(name string) error {
		return errors.Errorf("package name '%s' is longer than the maximum name length (%d): ", name, MAX_NAME_LEN)
	}
	for _, r := range name {
		err := ValidPackagePathChar(r)
		err := ValidPackageNameChar(r)
		if err != nil {
			return err
		}


@@ 121,7 138,7 @@ func ValidName(name string) error {

func ValidAddress(address string) error {
	for _, r := range address {
		err := ValidPackagePathChar(r)
		err := ValidPackageNameChar(r)
		if err != nil {
			return err
		}

M src/store/init.go => src/store/init.go +7 -0
@@ 7,6 7,7 @@ import (
	"path/filepath"

	"github.com/andrewchambers/hermes/extrasqlite"
	"github.com/andrewchambers/hermes/pkgs"
	"github.com/bvinc/go-sqlite-lite/sqlite3"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"


@@ 23,6 24,12 @@ type StoreInitConfigTemplate struct {
// InitStore creates a hermes store and initializes the database.
// InitStore will not overwrite a store that has already been initialized
func InitStore(storePath string, configTemplate StoreInitConfigTemplate) error {
	for _, r := range storePath {
		if err := pkgs.ValidStorePathChar(r); err != nil {
			return err
		}
	}

	err := os.MkdirAll(storePath, 0755)
	if err != nil {
		return errors.Wrap(err, "error while making store dir")

A tests/devtests/0012-pkgtree.test => tests/devtests/0012-pkgtree.test +32 -0
@@ 0,0 1,32 @@
#! /usr/bin/env bash

set -eu
cd "$( dirname "${BASH_SOURCE[0]}" )" 

. ./devtests.inc

mkdir a
echo "a = fetch(\"./a.txt\")"  > ./a/a.hpkg
echo a > ./a/a.txt
mkdir b
echo "b = fetch(\"./b.txt\")"  > ./b/b.hpkg
echo b > ./b/b.txt

cat << 'EOF' > ./t.hpkg
hpkgs = pkg_tree(
  ["a", fetch("./a")],
  ["b", "./b"],
  hash = "sha256:0839331cbf90c1df717d83658f64b0449c3de17e7c94cd0dc8c55d054c38320e"
)

load([hpkgs, "/a/a.hpkg"], "a")
load([hpkgs, "/b/b.hpkg"], "b")

p1 = a
p2 = b
EOF

hermes build ./t.hpkg --verbose -e p1
diff -u ./result/a.txt ./a/a.txt
hermes build ./t.hpkg --verbose -e p2
diff -u ./result/b.txt ./b/b.txt