~ach/hermes unlisted

b2599991aaddb8326be84178a18443d012bbb798 — Andrew Chambers 1 year, 1 month ago 4f42a96 master
Add pkg_tree builtin.
M src/cmd/builtins/fetch.go => src/cmd/builtins/fetch.go +1 -1
@@ 10,7 10,7 @@ import (
)

func onDiag(m *fetch.DiagnosticMessage) {
	_, _ = fmt.Fprintf(os.Stderr, "%s", m.Message)
	_, _ = fmt.Fprintf(os.Stderr, "%s\n", m.Message)
}

func fetchMain() {

M src/fetch/fetch.go => src/fetch/fetch.go +15 -3
@@ 5,6 5,7 @@ import (
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"



@@ 32,7 33,7 @@ func fetchLocal(u *url.URL) (io.ReadCloser, error) {

	f, err := os.Open(u.Path)
	if err != nil {
		return nil, errors.Wrapf(err, "error opening file at url '%s'\n", u)
		return nil, errors.Wrapf(err, "error opening file at url '%s'", u)
	}
	return f, nil
}


@@ 40,7 41,7 @@ func fetchLocal(u *url.URL) (io.ReadCloser, error) {
func fetchHttp(u *url.URL) (io.ReadCloser, error) {
	resp, err := http.Get(u.String())
	if err != nil {
		return nil, errors.Wrapf(err, "error fetching url  '%s'\n", u.String())
		return nil, errors.Wrapf(err, "error fetching url '%s'", u.String())
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		_ = resp.Body.Close()


@@ 49,13 50,24 @@ func fetchHttp(u *url.URL) (io.ReadCloser, error) {
	return resp.Body, nil
}

func fetchHpkg(u *url.URL) (io.ReadCloser, error) {
	remappedUrl := &url.URL{
		Scheme: "https",
		Host:   u.Host,
		Path:   path.Join(u.Path, "archive", u.Fragment+".tar.gz"),
	}
	return fetchHttp(remappedUrl)
}

func fetchUrl(u *url.URL) (io.ReadCloser, error) {
	switch u.Scheme {
	case "file":
		return fetchLocal(u)
	case "http", "https":
		return fetchHttp(u)
	case "hpkg":
		return fetchHpkg(u)
	default:
		return nil, errors.Errorf("don't know how to fetch '%s'\n", u.String())
		return nil, errors.Errorf("don't know how to fetch '%s'", u.String())
	}
}

M src/fetch/proto.go => src/fetch/proto.go +5 -5
@@ 107,7 107,7 @@ func handleFetchRequest(mirrorDB map[string][]string, workingDir string, c io.Re
		// closure is just so defers work.
		ok, err := func() (bool, error) {

			err := WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("trying url: %q\n", FilterCredentials(mirror, mirror.String()))})
			err := WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("trying url: %q", FilterCredentials(mirror, mirror.String()))})
			if err != nil {
				return false, err
			}


@@ 146,7 146,7 @@ func handleFetchRequest(mirrorDB map[string][]string, workingDir string, c io.Re
						if !ok {
							return nil
						}
						_ = WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("%d bytes downloaded\n", n)})
						_ = WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("%d bytes downloaded", n)})
					}
				}
			})


@@ 185,7 185,7 @@ func handleFetchRequest(mirrorDB map[string][]string, workingDir string, c io.Re
			// from just taking files.
			if req.Hash != "" || mirror.Scheme == "file" {
				if gotHash != req.Hash {
					_ = WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("bad download:\nexpected '%s'\ngot '%s'\n", req.Hash, gotHash)})
					_ = WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("bad download:\nexpected '%s'\ngot '%s'", req.Hash, gotHash)})
					return false, nil
				}
			}


@@ 195,7 195,7 @@ func handleFetchRequest(mirrorDB map[string][]string, workingDir string, c io.Re
				return false, err
			}

			err = WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("download ok, sending file '%s' to build environment\n", gotHash)})
			err = WritePacket(c, &DiagnosticMessage{Message: fmt.Sprintf("download ok, sending file '%s' to build environment", gotHash)})
			if err != nil {
				return false, err
			}


@@ 223,7 223,7 @@ func handleFetchRequest(mirrorDB map[string][]string, workingDir string, c io.Re

	}

	return WritePacket(c, &FetchError{Message: fmt.Sprintf("no mirrors successfully provided content.\n")})
	return WritePacket(c, &FetchError{Message: fmt.Sprintf("no mirrors successfully provided content.")})
}

func ServeConnection(mirrorDB map[string][]string, workingDir string, c net.Conn) error {

M src/pkgs/interp/builtins.go => src/pkgs/interp/builtins.go +101 -58
@@ 23,7 23,6 @@ func init() {
	BuiltinEnv = make(hscript.StringDict)
	BuiltinEnv["fetch"] = hscript.NewBuiltin("fetch", fetchBuiltin)
	BuiltinEnv["pkg_tree"] = hscript.NewBuiltin("pkg_tree", pkgTreeBuiltin)

	BuiltinEnv["builtins"] = Placeholder{"builtins"}
	BuiltinEnv["build_platform"] = &hscript.Module{
		Name: "build_platform",


@@ 45,7 44,7 @@ func init() {
// Untangling it would be a good excercise for anyone with time and some creativity.
// Try to pay attention to performance too, fetch is actually in a performance critical code path when
// it is used for loading in source code and patches.
func processAndValidateFetchUrl(fn *hscript.Builtin, mirrors *MirrorMap, currentDir string, hash, fetchUrl, file_name, unpack_type *string, unpack, unpack_unwrap *bool) error {
func processAndValidateFetchUrl(mirrors *MirrorMap, currentDir string, hash, fetchUrl, file_name, unpack_type *string, unpack, unpack_unwrap *bool) error {
	var parsedUrl *url.URL

	if strings.HasPrefix(*fetchUrl, ".") || strings.HasPrefix(*fetchUrl, "/") {


@@ 117,7 116,7 @@ func processAndValidateFetchUrl(fn *hscript.Builtin, mirrors *MirrorMap, current
	} else if *fetchUrl != "" {
		u, err := url.Parse(*fetchUrl)
		if err != nil {
			return errors.Wrapf(err, "unable to parse url: %q", fetchUrl)
			return errors.Wrapf(err, "unable to parse url %q", fetchUrl)
		}
		parsedUrl = u
	}


@@ 127,7 126,7 @@ func processAndValidateFetchUrl(fn *hscript.Builtin, mirrors *MirrorMap, current
	}

	if *hash == "" && *fetchUrl == "" {
		return errors.Errorf("%s requires either a hash or a url", fn.Name())
		return errors.Errorf("fetch requires either a hash or a url")
	}

	if *hash != "" && *fetchUrl != "" {


@@ 145,17 144,81 @@ func processAndValidateFetchUrl(fn *hscript.Builtin, mirrors *MirrorMap, current
	return nil
}

func addBuilderFragmentForUrlImport(mirrors *MirrorMap, currentDir, importPath, fetchUrl, filename string, builder *hscript.List) error { // no free variables
	fetchHash := ""
	unpack := true
	unpack_unwrap := true
	unpack_type := ""

	err := processAndValidateFetchUrl(mirrors, currentDir, &fetchHash, &fetchUrl, &filename, &unpack_type, &unpack, &unpack_unwrap)
	if err != nil {
		return err
	}

	urlArg := ""
	if fetchUrl != "" {
		urlArg = " " + strconv.Quote("--url="+fetchUrl)
	}

	hashArg := ""
	if fetchHash != "" {
		hashArg = " " + strconv.Quote("--hash="+fetchHash)
	}

	unpackTypeArg := ""
	if unpack_type != "" {
		unpackTypeArg = " " + strconv.Quote("--type="+unpack_type)
	}

	builder.Append(Placeholder{"builtins"})
	builder.Append(
		hscript.String(
			fmt.Sprintf(
				"/fetch%s%s %q\n",
				urlArg,
				hashArg,
				"--out="+filename,
			),
		),
	)
	builder.Append(Placeholder{"builtins"})
	builder.Append(
		hscript.String(
			fmt.Sprintf(
				"/unpack%s %q %q\n",
				unpackTypeArg,
				"--dest=${out}/"+importPath,
				filename,
			),
		),
	)

	return nil
}

func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tuple, kwargs []hscript.Tuple) (hscript.Value, error) {
	mirrors := thread.Local("mirror_db").(*MirrorMap)
	currentDir := thread.Local("current_dir").(string)

	fnname := fn.Name()
	contentHash := ""

	importSpecs := args
	err := hscript.UnpackArgs(fnname, hscript.Tuple{}, kwargs, "hash", &contentHash)
	if err != nil {
		return nil, err

	if len(kwargs) != 0 {
		err := hscript.UnpackArgs(fnname, hscript.Tuple{}, kwargs, "hash", &contentHash)
		if err != nil {
			return nil, err
		}
	} else {
		if len(importSpecs) <= 2 {
			return nil, errors.Errorf("%s: expects at least an import and an output hash", fnname)
		}
		contentHashHstr, ok := args[len(importSpecs)-1].(hscript.String)
		if !ok {
			return nil, errors.Errorf("%s: output hash must be a string", fnname)
		}
		contentHash = string(contentHashHstr)
		importSpecs = args[0 : len(importSpecs)-1]
	}

	builder := hscript.NewList([]hscript.Value{


@@ 170,6 233,30 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu

	for argIdx, imp := range importSpecs {
		switch imp := imp.(type) {
		case hscript.String:
			importSpec := string(imp)
			importSpecWithScheme := importSpec

			if !strings.HasPrefix(importSpecWithScheme, "hpkg://") {
				importSpecWithScheme = "hpkg://" + importSpec
			}

			importUrl, err := url.Parse(importSpecWithScheme)
			if err != nil {
				return nil, errors.Wrapf(err, "%s", fnname)
			}

			if importUrl.Fragment == "" {
				return nil, errors.Errorf("%s: empty #version specifier for import %q", fnname, importSpec)
			}

			importPath := importUrl.Host + importUrl.Path

			err = addBuilderFragmentForUrlImport(mirrors, currentDir, importPath, importUrl.String(), "pkgs.tgz", builder)
			if err != nil {
				return nil, errors.Wrapf(err, "%s", fnname)
			}

		case *hscript.List:
			if imp.Len() != 2 {
				return nil, malformedArgError(argIdx)


@@ 187,57 274,11 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu

			switch importContent := imp.Index(1).(type) {
			case hscript.String:
				// we are importing from a url string.
				fetchUrl := string(importContent)
				file_name := ""
				fetchHash := ""
				unpack := true
				unpack_unwrap := true
				unpack_type := ""

				err = processAndValidateFetchUrl(fn, mirrors, currentDir, &fetchHash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
				err := addBuilderFragmentForUrlImport(mirrors, currentDir, importPath, string(importContent), "", builder)
				if err != nil {
					return nil, err
				}

				urlArg := ""
				if fetchUrl != "" {
					urlArg = " " + strconv.Quote("--url="+fetchUrl)
				}

				hashArg := ""
				if fetchHash != "" {
					hashArg = " " + strconv.Quote("--hash="+fetchHash)
				}

				unpackTypeArg := ""
				if unpack_type != "" {
					unpackTypeArg = " " + strconv.Quote("--type="+unpack_type)
					return nil, errors.Wrapf(err, "%s", fnname)
				}

				builder.Append(Placeholder{"builtins"})
				builder.Append(
					hscript.String(
						fmt.Sprintf(
							"/fetch%s%s %q\n",
							urlArg,
							hashArg,
							"--out="+file_name,
						),
					),
				)
				builder.Append(Placeholder{"builtins"})
				builder.Append(
					hscript.String(
						fmt.Sprintf(
							"/unpack%s %q %q\n",
							unpackTypeArg,
							"--dest=${out}/"+importPath,
							file_name,
						),
					),
				)

			case *hscript.List, *hscript.Dict:
				// We are importing from a package.
				builder.Append(Placeholder{"builtins"})


@@ 251,6 292,8 @@ func pkgTreeBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tu
			default:
				return nil, malformedArgError(argIdx)
			}
		default:
			return nil, errors.Errorf("%s: positional arguments must match \"import-spec\" | [\"import-path\", \"url\"| pkg]", fnname)
		}
	}



@@ 286,9 329,9 @@ func fetchBuiltin(thread *hscript.Thread, fn *hscript.Builtin, args hscript.Tupl
		return nil, err
	}

	err = processAndValidateFetchUrl(fn, mirrors, currentDir, &hash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
	err = processAndValidateFetchUrl(mirrors, currentDir, &hash, &fetchUrl, &file_name, &unpack_type, &unpack, &unpack_unwrap)
	if err != nil {
		return nil, err
		return nil, errors.Wrapf(err, "%s", fn.Name())
	}

	urlArg := ""

M tests/devtests/0012-pkgtree.test => tests/devtests/0012-pkgtree.test +1 -1
@@ 16,7 16,7 @@ cat << 'EOF' > ./t.hpkg
hpkgs = pkg_tree(
  ["a", fetch("./a")],
  ["b", "./b"],
  hash = "sha256:0839331cbf90c1df717d83658f64b0449c3de17e7c94cd0dc8c55d054c38320e"
  "sha256:0839331cbf90c1df717d83658f64b0449c3de17e7c94cd0dc8c55d054c38320e"
)

load([hpkgs, "/a/a.hpkg"], "a")