From 3730d69cb28122d69f73b00fec18df6f7b46ad38 Mon Sep 17 00:00:00 2001 From: Noel Cower Date: Sat, 10 Mar 2018 13:04:30 -0800 Subject: [PATCH] Add per-file options, allow adding non-regular files - Allow reading non-regular files so they can be added to an archive. Non-regular files include, primarily, named pipes and devices, as well as stdin as a special case (via '-' -- /dev/stdin doesn't get special treatment). This is supported mainly by buffering those files into memory, since the actual size of those files when stat'd is not necessarily useful (especially for pipes). I'd like to try to avoid buffering things into memory in the future, but my particular use case right now isn't harmed by this. It might be worse for people using it in ways I'm not expecting. - File options are supported as a means of changing the properties of files as they're added to the archive. This allows you to synthesize some files, like directories and links, by doing, for example, '-:/etc:dir,owner=root,uid=0' or something along those lines. Currently, mtime, atime, and ctime can be modified using options, but if they're passed as integers, there's currently nothing to prevent integer overflow (and this isn't an error in Go, so it'll just silently overflow). This needs to be fixed, but also doesn't affect me right now. Change-Id: I9aca08609c1df5c57c18db5d13894f8c9510fb9e --- mtar.go | 379 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 349 insertions(+), 30 deletions(-) diff --git a/mtar.go b/mtar.go index 9ed163f..3c447ab 100644 --- a/mtar.go +++ b/mtar.go @@ -35,9 +35,9 @@ // // Writes a tar file to standard output. // -// FILE may be a regular filepath for a file, symlink, or directory. If -// FILE contains a ':', the text after the colon is the path to write to -// the tar file. For example, the following paths behave differently: +// FILE may be a filepath for a file, symlink, or directory. If FILE +// contains a ':', the text after the colon is the path to write to the tar +// file. For example, the following paths behave differently: // // SRC // Add file SRC to the tar file as-is. @@ -46,8 +46,40 @@ // SRC:DEST // Add file SRC as DEST to the tar file. // -// There is currently no SRC to accept standard input as a file (other -// than, for example, using /dev/stdin). +// To read a file from standard input, you can set '-' as the SRC. If no +// DEST is given for this, it will default to dev/stdin (relative). File +// permissions and ownership are taken from fd 1, so overriding them may be +// necessary. If the link or dir option is set, - can be used to synthesize +// a file entry. +// +// In the case of SRC: and SRC:DEST, you can also pass an additional :OPTS +// with either (i.e., SRC::OPTS or SRC:DEST:OPTS), where FLAGS are +// comma-separated strings that set options. The following options are +// available (all option names are case-sensitive): +// +// norec +// For directory entries, do not recursively add files from the +// directory. This will cause only the directory itself to appear as +// an entry. +// dir +// Force file to become a dir entry. Implies norec. +// link=LINK +// Force file to become a symlink pointing to LINK. +// uid=UID | owner=USERNAME +// Set the owner's uid and/or username for the file entry. +// gid=GID | group=GROUPNAME +// Set the gid and/or the group name for the file entry. +// mode=MODE +// Set the file mode to MODE (may be hex, octal, or an integer -- octal +// must begin with a 0, hex with 0x). +// mtime=TIME | atime=TIME | ctime=TIME +// Sets the mod time, access time, or changed time to TIME. May be an +// RFC3339 timestamp or an integer timestamp (since the Unix epoch) in +// seconds, milliseconds (>=12 digits), or microseconds (>=15 digits). +// +// Any whitespace preceding an option is trimmed. Whitespace is not trimmed +// before or after the '=' symbol for options that take values. Commas are +// not currently permitted inside options. // // In addition, options may be passed in the middle of file arguments to // control archive creation: @@ -73,6 +105,9 @@ package main // import "go.spiff.io/mtar" import ( "archive/tar" + "bytes" + "errors" + "fmt" "io" "log" "os" @@ -83,6 +118,7 @@ import ( "strconv" "strings" "syscall" + "time" ) type Args struct{ args []string } @@ -96,6 +132,7 @@ func (m Matcher) matches(s string) bool { return m.rx.MatchString(s) == m.want } +var startupTime = time.Now() var startupDir string var skipSrcGlobs []Matcher var skipDestGlobs []Matcher @@ -113,9 +150,9 @@ func usage() { Writes a tar file to standard output. -FILE may be a regular filepath for a file, symlink, or directory. If -FILE contains a ':', the text after the colon is the path to write to -the tar file. For example, the following paths behave differently: +FILE may be a filepath for a file, symlink, or directory. If FILE +contains a ':', the text after the colon is the path to write to the tar +file. For example, the following paths behave differently: SRC Add file SRC to the tar file as-is. @@ -124,8 +161,40 @@ the tar file. For example, the following paths behave differently: SRC:DEST Add file SRC as DEST to the tar file. -There is currently no SRC to accept standard input as a file (other -than, for example, using /dev/stdin). +To read a file from standard input, you can set '-' as the SRC. If no +DEST is given for this, it will default to dev/stdin (relative). File +permissions and ownership are taken from fd 1, so overriding them may be +necessary. If the link or dir option is set, - can be used to synthesize +a file entry. + +In the case of SRC: and SRC:DEST, you can also pass an additional :OPTS +with either (i.e., SRC::OPTS or SRC:DEST:OPTS), where FLAGS are +comma-separated strings that set options. The following options are +available (all option names are case-sensitive): + + norec + For directory entries, do not recursively add files from the + directory. This will cause only the directory itself to appear as + an entry. + dir + Force file to become a dir entry. Implies norec. + link=LINK + Force file to become a symlink pointing to LINK. + uid=UID | owner=USERNAME + Set the owner's uid and/or username for the file entry. + gid=GID | group=GROUPNAME + Set the gid and/or the group name for the file entry. + mode=MODE + Set the file mode to MODE (may be hex, octal, or an integer -- octal + must begin with a 0, hex with 0x). + mtime=TIME | atime=TIME | ctime=TIME + Sets the mod time, access time, or changed time to TIME. May be an + RFC3339 timestamp or an integer timestamp (since the Unix epoch) in + seconds, milliseconds (>=12 digits), or microseconds (>=15 digits). + +Any whitespace preceding an option is trimmed. Whitespace is not trimmed +before or after the '=' symbol for options that take values. Commas are +not currently permitted inside options. In addition, options may be passed in the middle of file arguments to control archive creation: @@ -165,6 +234,11 @@ func main() { w := tar.NewWriter(os.Stdout) defer func() { failOnError("error writing output", w.Close()) }() argv := Args{args: os.Args[1:]} + + if argv.args[0] == "--" { + argv.Shift() + } + for s, ok := argv.Shift(); ok; s, ok = argv.Shift() { switch { // Filter flags @@ -217,28 +291,46 @@ func main() { src, dest = s[:idx], s[idx+1:] } - addFile(w, src, dest, true) + var opts *FileOpts + if idx := strings.IndexByte(dest, ':'); idx > -1 { + opts, err = parseFileOptions(dest[idx+1:]) + failOnError("cannot parse options for "+src, err) + dest = dest[:idx] + } + + addFile(w, src, dest, opts, true) } } } -func addFile(w *tar.Writer, src, dest string, allowRecursive bool) { +func addFile(w *tar.Writer, src, dest string, opts *FileOpts, allowRecursive bool) { if shouldSkip(skipSrcGlobs, src) { return } - st, err := os.Lstat(src) + var needBuffer bool + var st os.FileInfo + var err error + + if src == "-" { + if dest == "" { + dest = "dev/stdin" + } + st, err = os.Stdin.Stat() + needBuffer = true + } else { + st, err = os.Lstat(src) + } + failOnError("add file: stat error", err) if dest == "" { - dest = src + dest = filepath.ToSlash(src) + if strings.HasPrefix(dest, "/") { + dest = path.Clean("." + dest) + } } - dest = path.Clean(filepath.ToSlash(dest)) - if strings.HasPrefix(dest, "/") { - dest = "." + dest - } else if !strings.HasPrefix("./", dest) { - dest = "./" + dest - } + if dest == ".." || strings.HasPrefix(dest, "../") { log.Fatal("add file: destination may not contain .. (", dest, ")") } @@ -267,6 +359,8 @@ func addFile(w *tar.Writer, src, dest string, allowRecursive bool) { switch { case st.Mode().IsRegular(): hdr.Size = st.Size() + case st.Mode()&(os.ModeCharDevice|os.ModeDevice|os.ModeNamedPipe) != 0: + needBuffer = true case st.IsDir(): hdr.Typeflag = tar.TypeDir hdr.Name = dest + "/" @@ -277,7 +371,7 @@ func addFile(w *tar.Writer, src, dest string, allowRecursive bool) { failOnError("cannot resolve symlink", err) hdr.Linkname = link default: - log.Print("skipping file: ", src, ": cannot add file with mode ", st.Mode().Perm()) + log.Print("skipping file: ", src, ": cannot add file") return } @@ -285,37 +379,66 @@ func addFile(w *tar.Writer, src, dest string, allowRecursive bool) { return } + opts.setHeaderFields(hdr) + + // Buffer input file if it's not a regular file + var r io.Reader + if needBuffer && hdr.Typeflag == tar.TypeReg { + var file *os.File + if src == "-" { + file = os.Stdin + } else { + file, err = os.Open(src) + failOnError("open error: "+src, err) + } + + var buf bytes.Buffer + _, err := io.Copy(&buf, file) + failOnError("unable to buffer "+src, err) + hdr.Size = int64(buf.Len()) + r = &buf + + if src != "-" { + failOnError("unable to close "+src, file.Close()) + } + } + failOnError("write header: "+hdr.Name, w.WriteHeader(hdr)) if st.Mode().IsDir() { - if allowRecursive { - addRecursive(w, src, dest) + if allowRecursive && opts.allowRecursive() { + addRecursive(w, src, dest, opts) } return } - if !st.Mode().IsRegular() { + if hdr.Typeflag != tar.TypeReg { return } - file, err := os.Open(src) - failOnError("read error: "+src, err) - defer file.Close() - n, err := io.Copy(w, file) + if r == nil { + file, err := os.Open(src) + failOnError("read error: "+src, err) + defer file.Close() + r = file + } + n, err := io.Copy(w, r) failOnError("copy error: "+src, err) if n != hdr.Size { log.Fatalf("copy error: size mismatch for %s: wrote %d, want %d", src, n, hdr.Size) } + + failOnError("flush error: "+src, w.Flush()) } -func addRecursive(w *tar.Writer, src, prefix string) { +func addRecursive(w *tar.Writer, src, prefix string, opts *FileOpts) { src = strings.TrimRight(src, "/") filepath.Walk(src, func(p string, info os.FileInfo, err error) error { if filepath.Clean(p) == filepath.Clean(src) || shouldSkip(skipSrcGlobs, p) { return nil } dest := path.Join(prefix, strings.TrimPrefix(p, src)) - addFile(w, p, dest, false) + addFile(w, p, dest, opts, false) return nil }) } @@ -359,3 +482,199 @@ func getUidGid(f os.FileInfo) (*user.User, *user.Group, bool) { } return u, g, true } + +type FileOpts struct { + noRecursive bool + + // exclusive: + dir bool + link string + + uid *int + username string + + gid *int + group string + + mode int64 + + mtime time.Time + atime time.Time + ctime time.Time +} + +func newFileOpts() *FileOpts { + return &FileOpts{} +} + +var timeLayouts = []string{ + time.RFC3339Nano, + // TODO: add additional layouts if needed +} + +func parseFileOptions(opts string) (*FileOpts, error) { + fields := strings.FieldsFunc(opts, isComma) + if len(fields) == 0 { + return nil, nil + } + + var err error + fo := newFileOpts() + for _, f := range fields { + f = strings.TrimSpace(f) + if f == "" { + continue + } + switch { + case f == "norec": + fo.noRecursive = true + case f == "dir": + if fo.link != "" { + return nil, fmt.Errorf("may not set dir with link=%s", fo.link) + } + fo.dir = true + fo.noRecursive = true + case strings.HasPrefix(f, "link="): + if fo.dir { + return nil, errors.New("may not set link with dir") + } + if fo.link = f[len("link="):]; fo.link == "" { + return nil, errors.New("may not set an empty link name") + } + case strings.HasPrefix(f, "uid="): + if uid, err := strconv.Atoi(f[len("uid="):]); err != nil { + return nil, fmt.Errorf("invalid uid: %v", err) + } else { + fo.uid = &uid + } + case strings.HasPrefix(f, "gid="): + if gid, err := strconv.Atoi(f[len("gid="):]); err != nil { + return nil, fmt.Errorf("invalid gid: %v", err) + } else { + fo.gid = &gid + } + case strings.HasPrefix(f, "owner="): + if fo.username = f[len("owner="):]; fo.username == "" { + return nil, errors.New("may not set an empty username for an owner") + } + case strings.HasPrefix(f, "group="): + if fo.group = f[len("group="):]; fo.group == "" { + return nil, errors.New("may not set an empty group name") + } + case strings.HasPrefix(f, "mode="): + if fo.mode, err = strconv.ParseInt(f[len("mode="):], 0, 64); err != nil { + return nil, fmt.Errorf("invalid mode: %v", err) + } else if fo.mode == 0 { + return nil, errors.New("invalid mode: may not be 0") + } + case strings.HasPrefix(f, "mtime=") || strings.HasPrefix(f, "atime=") || strings.HasPrefix(f, "ctime="): + var tp *time.Time + switch f[0] { + case 'm': + tp = &fo.mtime + case 'a': + tp = &fo.atime + case 'c': + tp = &fo.ctime + } + *tp = time.Time{} + + ts := f[len("mtime="):] + if ts == "now" { + *tp = startupTime + continue + } + + for _, layout := range timeLayouts { + var t time.Time + if t, err = time.Parse(layout, ts); err == nil { + *tp = t + break + } + } + + // Integer timestamp + if err != nil { + var ti int64 + ti, err = strconv.ParseInt(ts, 10, 64) + if err != nil { + goto timeFailure + } + // TODO: handle integer overflow + if len(ts) >= 15 { // microseconds + dur := time.Duration(ti) * time.Microsecond + *tp = time.Unix(int64(dur/time.Second), int64(dur%time.Second)) + } else if len(ts) >= 12 { // milliseconds + dur := time.Duration(ti) * time.Millisecond + *tp = time.Unix(int64(dur/time.Second), int64(dur%time.Second)) + } else { // seconds + *tp = time.Unix(ti, 0) + } + } + timeFailure: + if tp.IsZero() { + return nil, fmt.Errorf("invalid %s: %q", f[:len("mtime")], ts) + } + default: + return nil, fmt.Errorf("unexpected option: %q", f) + } + } + + return fo, nil +} + +func (f *FileOpts) allowRecursive() bool { + return f == nil || !f.noRecursive +} + +func (f *FileOpts) setHeaderFields(hdr *tar.Header) { + if f == nil { + return + } + + if f.uid != nil { + hdr.Uid = *f.uid + } + if f.gid != nil { + hdr.Gid = *f.gid + } + + if f.username != "" { + hdr.Uname = f.username + } + if f.group != "" { + hdr.Gname = f.group + } + + if f.mode != 0 { + hdr.Mode = f.mode + } + + if f.dir { + hdr.Typeflag = tar.TypeDir + hdr.Linkname = "" + hdr.Size = 0 + if !strings.HasSuffix(hdr.Name, "/") { + hdr.Name += "/" + } + } else if f.link != "" { + hdr.Linkname = f.link + hdr.Typeflag = tar.TypeSymlink + } + + if !f.mtime.IsZero() { + hdr.ModTime = f.mtime + } + + if !f.atime.IsZero() { + hdr.AccessTime = f.atime + } + + if !f.ctime.IsZero() { + hdr.ChangeTime = f.ctime + } +} + +func isComma(r rune) bool { + return r == ',' +} -- 2.45.2