M TODO.txt => TODO.txt +1 -0
@@ 58,6 58,7 @@ It seems useful to see it regardless. This is a maybe as it is bad for performan
Possibly via a builtin command that can be run. @{builtins}/debug
- Interactive debug should perhaps be automatically entered on failure?.
- Make C++ runtime it's own package somehow?
+- Error messages for bad loads.
Maybe:
M csrc/hermes-linux-namespace-sandbox.c => csrc/hermes-linux-namespace-sandbox.c +9 -0
@@ 383,6 383,15 @@ static int post_clone(void *argv) {
NULL,
};
+ if (verbose) {
+ fprintf(stderr, "execve: [");
+ char **arg = builder;
+ while (*arg) {
+ fprintf(stderr, " %s", *arg);
+ arg += 1;
+ }
+ fprintf(stderr, " ]\n");
+ }
execve(builder[0], builder, env);
die("execve: %s", strerror(errno));
return 1;
M do => do +4 -6
@@ 22,9 22,9 @@ vcmd () {
build_go_bin () {
unset GOPATH
out="$(pwd)/$2/$1"
- pushd "src/cmd/$1"
- vcmd go build $HERMES_GO_EXTRA_BUILD_FLAGS -mod=vendor -tags "netgo" -ldflags "-extldflags -static" -o "$out"
- popd
+ pushd "src/cmd/$1" > /dev/null
+ vcmd go build $HERMES_GO_EXTRA_BUILD_FLAGS -mod=vendor -ldflags "-linkmode=external -extldflags -static" -o "$out"
+ popd > /dev/null
}
test -f "config.inc" || fail "please run ./configure first."
@@ 44,7 44,7 @@ case "$1" in
build_go_bin builtins build/builtins
- for cmd in fetch unpack writefile runcmds
+ for cmd in fetch unpack writefile runcmds mkdir copy
do
(cd "build/builtins" && ln -s ./builtins "$cmd")
done
@@ 59,9 59,7 @@ EOF
build_go_bin hermes build/sysroot/bin
build_go_bin hermes-pkgstore build/sysroot/libexec/hermes
-
${CC} ${CFLAGS} ${LDFLAGS} -o build/sysroot/libexec/hermes/hermes-linux-namespace-sandbox csrc/hermes-linux-namespace-sandbox.c
-
;;
install)
A src/cmd/builtins/copy.go => src/cmd/builtins/copy.go +128 -0
@@ 0,0 1,128 @@
+package main
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "regexp"
+
+ flag "github.com/spf13/pflag"
+)
+
+var skipReList []*regexp.Regexp
+
+func copy(src, dest string) error {
+ info, err := os.Lstat(src)
+ if err != nil {
+ return err
+ }
+ return copy2(src, dest, info)
+}
+
+func copy2(src, dest string, info os.FileInfo) error {
+
+ for _, skipRe := range skipReList {
+ if skipRe.MatchString(src) {
+ return nil
+ }
+ }
+
+ if info.Mode()&os.ModeSymlink != 0 {
+
+ src, err := os.Readlink(src)
+ if err != nil {
+ return err
+ }
+
+ return os.Symlink(src, dest)
+ } else if info.IsDir() {
+
+ originalMode := info.Mode()
+
+ err := os.MkdirAll(dest, os.FileMode(0755))
+ if err != nil {
+ return err
+ }
+
+ contents, err := ioutil.ReadDir(src)
+ if err != nil {
+ return err
+ }
+
+ for _, content := range contents {
+ cs, cd := filepath.Join(src, content.Name()), filepath.Join(dest, content.Name())
+ err := copy2(cs, cd, content)
+ if err != nil {
+ return err
+ }
+ }
+
+ err = os.Chmod(dest, originalMode)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ } else if info.Mode().IsRegular() {
+
+ err := os.MkdirAll(filepath.Dir(dest), os.ModePerm)
+ if err != nil {
+ return err
+ }
+
+ f, err := os.Create(dest)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ err = os.Chmod(f.Name(), info.Mode())
+ if err != nil {
+ return err
+ }
+
+ s, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer s.Close()
+
+ _, err = io.Copy(f, s)
+ if err != nil {
+ return err
+ }
+
+ err = f.Close()
+ if err != nil {
+ return err
+ }
+
+ return nil
+ } else {
+ return fmt.Errorf("don't know how to copy path %q", src)
+ }
+}
+
+func copyMain() {
+ skip := flag.StringArray("skip", []string{}, "A list of regular expressions matched against files to skip.")
+ flag.Parse()
+
+ for _, s := range *skip {
+ r, err := regexp.Compile(s)
+ if err != nil {
+ die("Error compiling regex %q: %s\n", s, err)
+ }
+ skipReList = append(skipReList, r)
+ }
+
+ if len(flag.Args()) != 2 {
+ die("Expected SRC and DEST.\n")
+ }
+
+ src := flag.Args()[0]
+ dst := flag.Args()[1]
+
+ check(copy(src, dst))
+}
M src/cmd/builtins/main.go => src/cmd/builtins/main.go +5 -1
@@ 36,7 36,11 @@ func main() {
runcmdsMain()
case "writefile":
writefileMain()
+ case "mkdir":
+ mkdirMain()
+ case "copy":
+ copyMain()
default:
- die("Please run with argv[0] as as one of \"builtins\", \"fetch\", \"runcmds\", \"writefile\" ,or \"unpack\", got %q.\n", prog)
+ die("Please run with argv[0] as as one of \"builtins\", \"fetch\", \"mkdir\", \"copy\", \"runcmds\", \"writefile\", or \"unpack\", got %q.\n", prog)
}
}
A src/cmd/builtins/mkdir.go => src/cmd/builtins/mkdir.go +26 -0
@@ 0,0 1,26 @@
+package main
+
+import (
+ "os"
+
+ flag "github.com/spf13/pflag"
+)
+
+func mkdirMain() {
+ parents := flag.BoolP("parents", "p", false, "no error if existing, make parent directories as needed.")
+
+ flag.Parse()
+
+ if len(flag.Args()) != 1 {
+ die("Expected a single argument.\n")
+ }
+
+ mode := os.FileMode(0755)
+
+ dir := flag.Args()[0]
+ if *parents {
+ check(os.MkdirAll(dir, mode))
+ } else {
+ check(os.Mkdir(dir, mode))
+ }
+}
M src/cmd/builtins/unpack.go => src/cmd/builtins/unpack.go +1 -0
@@ 21,6 21,7 @@ func untar(r io.Reader, dest string, noUnwrap bool) {
dir, err := filepath.Abs(dest)
check(err)
+ check(os.MkdirAll(dir, 0755))
topLevelCount := 0
lastTopLevelItem := ""
M src/cmd/hermes/fetchserver.go => src/cmd/hermes/fetchserver.go +13 -0
@@ 4,9 4,11 @@ import (
"fmt"
"net"
"os"
+ "os/signal"
"github.com/andrewchambers/hermes/fetch"
flag "github.com/spf13/pflag"
+ "golang.org/x/sys/unix"
"olympos.io/encoding/edn"
)
@@ 56,6 58,16 @@ func fetchServerMain() {
}
}
+ sigs := make(chan os.Signal, 1)
+ signal.Notify(sigs, os.Interrupt, unix.SIGTERM, unix.SIGHUP)
+ go func() {
+ <-sigs
+ // XXX The fetch server doesn't have much state we need to cleanup.
+ // we should use a context anyway however.
+ _ = os.Remove(opts.Socket)
+ os.Exit(0)
+ }()
+
l, err := net.Listen("unix", opts.Socket)
if err != nil {
die("Error listening on socket: %s\n", err)
@@ 63,5 75,6 @@ func fetchServerMain() {
_, err = fmt.Println("up")
err = fetch.ServeRequests(l, mirrorDB, opts.WorkDir)
+ _ = os.Remove(opts.Socket)
die("Server exiting: %s\n", err)
}
M src/pkgs/interp/builtins.go => src/pkgs/interp/builtins.go +35 -10
@@ 149,9 149,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
mirrors := thread.Local("mirror_db").(*MirrorMap)
currentDir := thread.Local("current_dir").(string)
- hash := ""
+ fnname := fn.Name()
+ contentHash := ""
- err := hscript.UnpackArgs(fn.Name(), args, kwargs, "hash", &hash)
+ importSpecs := args
+ err := hscript.UnpackArgs(fnname, hscript.Tuple{}, kwargs, "hash", &contentHash)
if err != nil {
return nil, err
}
@@ 163,10 165,10 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
})
malformedArgError := func(idx int) error {
- return errors.Errorf("argument %d should be of the form [\"importpath\", \"url\" | package]", idx)
+ return errors.Errorf("%s: argument %d should be of the form [\"importpath\", \"url\" | package]", fnname, idx)
}
- for argIdx, imp := range args {
+ for argIdx, imp := range importSpecs {
switch imp := imp.(type) {
case *hscript.List:
if imp.Len() != 2 {
@@ 188,21 190,38 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
// we are importing from a url string.
fetchUrl := string(importContent)
file_name := ""
+ fetchHash := ""
unpack := true
unpack_unwrap := true
unpack_type := ""
- err = processAndValidateFetchUrl(fn, mirrors, currentDir, &hash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
+ err = processAndValidateFetchUrl(fn, mirrors, currentDir, &fetchHash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
if err != nil {
return nil, err
}
+ urlArg := ""
+ if fetchUrl != "" {
+ urlArg = " " + strconv.Quote("--url="+fetchUrl)
+ }
+
+ hashArg := ""
+ if fetchHash != "" {
+ hashArg = " " + strconv.Quote("--hash="+fetchHash)
+ }
+
+ unpackTypeArg := ""
+ if unpack_type != "" {
+ unpackTypeArg = " " + strconv.Quote("--type="+unpack_type)
+ }
+
builder.Append(Placeholder{"builtins"})
builder.Append(
hscript.String(
fmt.Sprintf(
- "/fetch %q %q\n",
- "--url="+fetchUrl,
+ "/fetch%s%s %q\n",
+ urlArg,
+ hashArg,
"--out="+file_name,
),
),
@@ 211,7 230,8 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
builder.Append(
hscript.String(
fmt.Sprintf(
- "/unpack %q %q\n",
+ "/unpack%s %q %q\n",
+ unpackTypeArg,
"--dest=${out}/"+importPath,
file_name,
),
@@ 221,7 241,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
case *hscript.List, *hscript.Dict:
// We are importing from a package.
builder.Append(Placeholder{"builtins"})
- builder.Append(hscript.String("/symlink \""))
+ builder.Append(hscript.String(fmt.Sprintf("/mkdir -p %q\n", "${out}/"+filepath.Dir(importPath))))
+ builder.Append(Placeholder{"builtins"})
+ // We choose to copy instead of link, so the package content hash does not depend on the package path.
+ // when copying we also skip the .hermes_pkg file as that contains the store path.
+ builder.Append(hscript.String("/copy --skip \"[^\\.]+\\.hermes_pkg$\" \""))
builder.Append(importContent)
builder.Append(hscript.String(fmt.Sprintf("\" %q\n", "${out}/"+importPath)))
default:
@@ 231,8 255,9 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
}
d := &hscript.Dict{}
+ d.SetKey(hscript.String("name"), hscript.String("pkg-tree"))
d.SetKey(hscript.String("builder"), builder)
- d.SetKey(hscript.String("content"), hscript.String(hash))
+ d.SetKey(hscript.String("content"), hscript.String(contentHash))
return d, nil
}
M src/pkgs/pkg.go => src/pkgs/pkg.go +25 -8
@@ 92,15 92,32 @@ func DecodeAddressFromPath(storePath, pkgPath string) (string, error) {
return pkgHashAndName, nil
}
-func ValidPackagePathChar(r rune) error {
+// XXX whitelist better?
+func ValidStorePathChar(r rune) error {
switch r {
case '\r', '\n':
- return errors.New("package paths must not contain new lines")
+ return errors.New("store paths must not contain new lines")
case '"', '\'':
- return errors.New("package paths must not contain quotations")
- case '/', '\\':
- return errors.New("package paths must not contain '\\' or '/'")
- // XXX It may be useful to disallow whitespace so package expressions.
+ return errors.New("store paths must not contain quotations")
+ case '\\', '$':
+ return errors.Errorf("store paths must not contain %v", r)
+ case ' ', '\t':
+ return errors.New("store paths must not contain whitespace")
+ }
+ return nil
+}
+
+// XXX whitelist better?
+func ValidPackageNameChar(r rune) error {
+ switch r {
+ case '\r', '\n':
+ return errors.New("package names must not contain new lines")
+ case '"', '\'':
+ return errors.New("package names must not contain quotations")
+ case '/', '\\', '$':
+ return errors.Errorf("package names must not contain %v", r)
+ case ' ', '\t':
+ return errors.New("package names must not contain whitespace")
}
return nil
}
@@ 111,7 128,7 @@ func ValidName(name string) error {
return errors.Errorf("package name '%s' is longer than the maximum name length (%d): ", name, MAX_NAME_LEN)
}
for _, r := range name {
- err := ValidPackagePathChar(r)
+ err := ValidPackageNameChar(r)
if err != nil {
return err
}
@@ 121,7 138,7 @@ func ValidName(name string) error {
func ValidAddress(address string) error {
for _, r := range address {
- err := ValidPackagePathChar(r)
+ err := ValidPackageNameChar(r)
if err != nil {
return err
}
M src/store/init.go => src/store/init.go +7 -0
@@ 7,6 7,7 @@ import (
"path/filepath"
"github.com/andrewchambers/hermes/extrasqlite"
+ "github.com/andrewchambers/hermes/pkgs"
"github.com/bvinc/go-sqlite-lite/sqlite3"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
@@ 23,6 24,12 @@ type StoreInitConfigTemplate struct {
// InitStore creates a hermes store and initializes the database.
// InitStore will not overwrite a store that has already been initialized
func InitStore(storePath string, configTemplate StoreInitConfigTemplate) error {
+ for _, r := range storePath {
+ if err := pkgs.ValidStorePathChar(r); err != nil {
+ return err
+ }
+ }
+
err := os.MkdirAll(storePath, 0755)
if err != nil {
return errors.Wrap(err, "error while making store dir")
A tests/devtests/0012-pkgtree.test => tests/devtests/0012-pkgtree.test +32 -0
@@ 0,0 1,32 @@
+#! /usr/bin/env bash
+
+set -eu
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+
+. ./devtests.inc
+
+mkdir a
+echo "a = fetch(\"./a.txt\")" > ./a/a.hpkg
+echo a > ./a/a.txt
+mkdir b
+echo "b = fetch(\"./b.txt\")" > ./b/b.hpkg
+echo b > ./b/b.txt
+
+cat << 'EOF' > ./t.hpkg
+hpkgs = pkg_tree(
+ ["a", fetch("./a")],
+ ["b", "./b"],
+ hash = "sha256:0839331cbf90c1df717d83658f64b0449c3de17e7c94cd0dc8c55d054c38320e"
+)
+
+load([hpkgs, "/a/a.hpkg"], "a")
+load([hpkgs, "/b/b.hpkg"], "b")
+
+p1 = a
+p2 = b
+EOF
+
+hermes build ./t.hpkg --verbose -e p1
+diff -u ./result/a.txt ./a/a.txt
+hermes build ./t.hpkg --verbose -e p2
+diff -u ./result/b.txt ./b/b.txt