~welt/murse

6a4e221341d604abb308d8c951c659d91e216327 — welt 6 months ago db85e43 toast
Initial work on TVS support, and toaster.
16 files changed, 763 insertions(+), 798 deletions(-)

A client.go
A concurrent.go
D crypto.go
D download.go
M files.go
M flags.go
M go.mod
M go.sum
D jobs.go
D local.go
M main.go
D sql.go
A toaster.go
A upgrade.go
A util.go
A versioning.go
A client.go => client.go +86 -0
@@ 0,0 1,86 @@
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
)

type Client struct {
	repo string
	h    *http.Client
}

func NewClient(repo string) *Client {
	return &Client{
		repo: repo,
		h:    &http.Client{},
	}
}

func (c *Client) get(url string) (io.ReadCloser, error) {
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}

	req.Header.Add("User-Agent", "murse/0.0.0")

	resp, err := c.h.Do(req)
	if err != nil {
		return resp.Body, err
	}

	if resp.StatusCode != 200 {
		fmt.Println(url)
		return nil, errors.New("got status code " + strconv.Itoa(resp.StatusCode))
	}

	return resp.Body, nil
}

func (c *Client) getRead(url string) ([]byte, error) {
	var bytes []byte
	body, err := c.get(url)
	if err != nil {
		return bytes, err
	}

	defer body.Close()

	return io.ReadAll(body)
}

func (c *Client) GetRevision(v int) ([]Change, error) {
	var changes []Change
	resp, err := c.get(c.repo + "revisions/" + strconv.Itoa(v))
	if err != nil {
		return changes, err
	}

	defer resp.Close()

	err = json.NewDecoder(resp).Decode(&changes)
	if err != nil {
		return changes, err
	}

	return changes, err
}

func (c *Client) GetObject(object string) (io.ReadCloser, error) {
	return c.get(c.repo + "objects/" + object)
}

func (c *Client) GetLatestRevision() (int, error) {
	b, err := c.getRead(c.repo + "revisions/latest")
	if err != nil {
		return -1, err
	}

	return strconv.Atoi(strings.TrimSpace(string(b)))
}

A concurrent.go => concurrent.go +51 -0
@@ 0,0 1,51 @@
package main

import "context"

type Pool struct {
	Errch  chan error
	Jobch  chan func() error
	donech chan bool
	num    int
	ctx    context.Context
	ctxf   context.CancelFunc
}

func NewPool(workers int) *Pool {
	pool := &Pool{
		Errch:  make(chan error),
		Jobch:  make(chan func() error),
		donech: make(chan bool),
		num:    workers,
	}

	pool.ctx, pool.ctxf = context.WithCancel(context.Background())

	for i := 1; i != workers; i++ {
		go func() {
			for {
				select {
				case job := <-pool.Jobch:
					err := job()
					if err != nil {
						go func() { pool.Errch <- err }()
					}

				case <-pool.ctx.Done():
					pool.donech <- true
					return
				}
			}
		}()
	}

	return pool

}

func (p *Pool) Stop() {
	p.ctxf()
	for i := 1; i != p.num; i++ {
		_ = p.donech
	}
}

D crypto.go => crypto.go +0 -85
@@ 1,85 0,0 @@
package main

import (
	"crypto"
	"crypto/rsa"
	"crypto/sha512"
	"crypto/x509"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"log"
)

func init() {
	k, err := getKey()
	if err != nil {
		log.Fatal(err)
	}

	publicKey = k
}

func compareBytesHashSum(hash string, bytes *[]byte) (bool, error) {
	hdc, err := hex.DecodeString(hash)
	if err != nil {
		fmt.Println(err)
		return false, nil
	}

	var ha [48]byte
	copy(ha[:], hdc) // convert slice to array for hash checking
	hfb := sha512.Sum384(*bytes)

	if hfb == ha {
		return true, nil
	}
	return false, nil
}

func base16ToHash(hash string) (*[48]byte, error) {
	hdc, err := hex.DecodeString(hash)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	var ha [48]byte
	copy(ha[:], hdc) // convert slice to array

	return &ha, nil
}

func verifySig(sig *[]byte, bytes *[]byte) bool {
	hash := sha512.Sum384(*bytes)
	err := rsa.VerifyPKCS1v15(publicKey, crypto.SHA384, hash[:], *sig)
	if err != nil {
		fmt.Println(err)
		return false
	}

	return true
}

func parseKey(b *[]byte) (*rsa.PublicKey, error) {
	pp /* heh */, _ := pem.Decode(*b)
	if pp == nil {
		return nil, errors.New("invalid key")
	}
	if pp.Type != "PUBLIC KEY" {
		return nil, errors.New("not a proper public key")
	}

	pk, err := x509.ParsePKIXPublicKey(pp.Bytes)
	if err != nil {
		return nil, errors.New("failed to parse key")
	}

	var pkt *rsa.PublicKey
	var ok bool
	if pkt, ok = pk.(*rsa.PublicKey); !ok {
		return nil, err
	}
	return pkt, nil
}

D download.go => download.go +0 -112
@@ 1,112 0,0 @@
package main

import (
	"crypto/rsa"
	"crypto/sha512"
	"errors"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"

	"github.com/gabriel-vasile/mimetype"
	"github.com/klauspost/compress/zstd"
)

func downloadDevKey() (*rsa.PublicKey, error) {
	b, err := get("https://raw.githubusercontent.com/int-72h/ofatomic/main/ofatomic/ofpublic.pem")
	if err != nil {
		return nil, err
	}

	k, err := parseKey(b)
	if err != nil {
		return nil, err
	}

	return k, nil
}

func get(url string) (*[]byte, error) {
	r, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	defer r.Body.Close()
	b, err := io.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}

	if mimetype.Detect(b).String() == "application/zstd" {
		d, _ := zstd.NewReader(nil)
		defer d.Close()
		bd, err := d.DecodeAll(b, nil)
		if err != nil {
			return nil, err
		}

		defer d.Close()

		return &bd, nil
	}
	return &b, nil
}

func downloadAndVerify(path string, checksum string, lchecksum *string) error {
	ff := filepath.Join(opt.Directory, path)
	if _, err := os.Stat(ff); !os.IsNotExist(err) {
		b, err := os.ReadFile(ff)
		if err != nil {
			return err
		}

		h := sha512.Sum384(b)

		if isProtected(path) && !install && !opt.OverwriteProtected && lchecksum != nil {
			dh, err := base16ToHash(*lchecksum) // <^ if protected and modified
			if err != nil {
				return err
			}

			if *dh == h {
				log.Println("File is protected, will not overwrite: " + ff)
				return nil
			}
		}

		dh, err := base16ToHash(checksum)
		if err != nil {
			return err
		}
		if *dh == h {
			log.Println("File exists and matches hash, skipping: " + ff)
			return nil
		}
	}

	log.Println("Downloading: " + ff)

	err := os.MkdirAll(filepath.Dir(ff), 0777)
	if err != nil && !errors.Is(err, os.ErrExist) {
		return err
	}

	b, err := get(opt.URL + "/" + path)
	if err != nil {
		return err
	}

	t, err := compareBytesHashSum(checksum, b)
	if err != nil {
		return err
	}
	if !t {
		return errors.New("file failed integrity check:" + path)
	}

	os.WriteFile(ff, *b, 0666)
	return nil
}

M files.go => files.go +8 -76
@@ 1,89 1,21 @@
package main

import (
	"crypto/rsa"
	"database/sql"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

func init() {
	cd, err := os.UserCacheDir()
	if err != nil {
		log.Fatal(err)
	}

	if !fileExists(cd) || !fileExists(filepath.Join(cd, "murse")) {
		err = os.MkdirAll(filepath.Join(cd, "murse"), 0777)
		if err != nil {
			log.Fatal(err)
		}
	}
}

func getKey() (*rsa.PublicKey, error) {
	cd, err := os.UserCacheDir()
	if err != nil {
		return nil, err
	}

	f := filepath.Join(cd, "murse", "public.key")
	if fileExists(f) {
		b, err := os.ReadFile(f)
		if err != nil {
			return nil, err
		}

		k, err := parseKey(&b)
		if err != nil {
			return nil, err
		}

		return k, nil
func getInstalledRevision(path string) (int, error) {
	rev := -1
	bytes, err := os.ReadFile(filepath.Join(filepath.FromSlash(path), ".revision"))
	if os.IsNotExist(err) {
		return rev, nil
	}

	k, err := downloadDevKey()
	if err != nil {
		return nil, err
		return rev, err
	}

	return k, nil
}

// This will return the files required between two given versions
func getRevisedFiles(db *sql.DB, lrev int, nrev int) (map[string]string, error) {
	rf := make(map[string]string)
	for i := lrev; i != nrev+1; i++ {
		mp, err := filesFromSQLite(db, i)
		if err != nil {
			return nil, err
		}

		for k, v := range mp {
			rf[k] = v
		}
	}
	return rf, nil
}

func isProtected(file string) bool {
	if strings.HasPrefix(file, "cfg/") ||
		strings.HasSuffix(file, ".cfg") ||
		strings.HasSuffix(file, ".dem") ||
		strings.HasPrefix(file, "custom") ||
		strings.HasPrefix(file, "downloads") ||
		strings.HasSuffix(file, "cache") {
		return true
	}
	return false
}

func fileExists(file string) bool {
	_, err := os.Stat(file)
	if !os.IsNotExist(err) || err == nil {
		return true
	}
	return false
	return strconv.Atoi(strings.TrimSpace(string(bytes)))
}

M flags.go => flags.go +0 -57
@@ 1,58 1,1 @@
package main

import (
	"log"
	"os"
	"path/filepath"
	"runtime"

	"github.com/jessevdk/go-flags"
)

var opt struct {
	Threads            int    `short:"t" short:"n" default:"0" description:"Number of threads to use. Don't touch this unless you know what you're doing. 0 = auto."`
	Directory          string `short:"d" short:"p" default:"./open_fortress" description:"Directory to download to or modify. Will be created if it doesn't already exist."`
	URL                string `short:"u" default:"https://svn.openfortress.fun/launcher/files/" description:"The URL for the launcher files."`
	OverwriteProtected bool   `short:"x" long:"cfg-overwrite" description:"Overwrite protected files (such as demos, modified .cfg files)."`
	IgnoreLockfile     bool   `short:"b" description:"Ignore the lockfile, in case the program was previously ungracefully killed."`
	// Backward compatibility flags
	DisableSigning bool `short:"z" long:"disable-signing" description:"Disable signature checks. Note that unlike ofatomic, we only check the file database's signature, so this should have no performance impact during file download."`
	DisableHashing bool `short:"#" long:"disable-hashing" description:"This is a dummy flag meant for compatability purposes."`
}

func init() {
	_, err := flags.Parse(&opt)
	if err != nil {
		os.Exit(1)
	}

	lmd = filepath.Join(opt.Directory, ".murse", "local.json")
	lfd = filepath.Join(opt.Directory, ".murse", "lockfile")
}

func init() {
	// We subtract one from the CPU count to leave thread(s) for the OS.
	// We set it to one on tri-core-or-less systems so pablo.gonzales2003's
	// Gateway netbook doesn't burn to a crisp.

	// We limit the automatic count to 6 so we don't burn OF's servers
	// to a crisp as well.

	if opt.Threads < 1 {
		switch {
		case runtime.NumCPU() > 6:
			opt.Threads = 6
		case runtime.NumCPU() < 3:
			opt.Threads = 1
		}
	}

	if opt.Threads > 6 {
		log.Println("!")
		log.Printf("WARNING: You are using %v threads. ", opt.Threads)
		log.Println("Please practice proper netiquette and make sure the")
		log.Println("place you're fetching from is okay with that many")
		log.Println("concurrent downloads going at once.")
		log.Println("!")
	}
}

M go.mod => go.mod +3 -6
@@ 1,11 1,8 @@
module spiderden.net/go/murse

go 1.16
go 1.18

require (
	github.com/gabriel-vasile/mimetype v1.3.0
	github.com/gammazero/workerpool v1.1.2
	github.com/jessevdk/go-flags v1.5.0
	github.com/klauspost/compress v1.12.3
	modernc.org/sqlite v1.10.7
	github.com/bwmarrin/snowflake v0.3.0
	github.com/integrii/flaggy v1.5.2
)

M go.sum => go.sum +6 -88
@@ 1,89 1,7 @@
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/gabriel-vasile/mimetype v1.3.0 h1:4YOHITFLyYwF+iqG0ybSLGArRItynpfwdlWRmJnd75E=
github.com/gabriel-vasile/mimetype v1.3.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
github.com/gammazero/deque v0.1.0 h1:f9LnNmq66VDeuAlSAapemq/U7hJ2jpIWa4c09q8Dlik=
github.com/gammazero/deque v0.1.0/go.mod h1:KQw7vFau1hHuM8xmI9RbgKFbAsQFWmBpqQ2KenFLk6M=
github.com/gammazero/workerpool v1.1.2 h1:vuioDQbgrz4HoaCi2q1HLlOXdpbap5AET7xu5/qj87g=
github.com/gammazero/workerpool v1.1.2/go.mod h1:UelbXcO0zCIGFcufcirHhq2/xtLXJdQ29qZNlXG9OjQ=
github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210505024714-0287a6fb4125 h1:Ugb8sMTWuWRC3+sz5WeN/4kejDx9BvIwnPUiJBjJE+8=
golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs=
golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/integrii/flaggy v1.5.2 h1:bWV20MQEngo4hWhno3i5Z9ISPxLPKj9NOGNwTWb/8IQ=
github.com/integrii/flaggy v1.5.2/go.mod h1:dO13u7SYuhk910nayCJ+s1DeAAGC1THCMj1uSFmwtQ8=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
modernc.org/cc/v3 v3.32.4/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878=
modernc.org/cc/v3 v3.33.5 h1:gfsIOmcv80EelyQyOHn/Xhlzex8xunhQxWiJRMYmPrI=
modernc.org/cc/v3 v3.33.5/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878=
modernc.org/ccgo/v3 v3.9.2/go.mod h1:gnJpy6NIVqkETT+L5zPsQFj7L2kkhfPMzOghRNv/CFo=
modernc.org/ccgo/v3 v3.9.4 h1:mt2+HyTZKxva27O6T4C9//0xiNQ/MornL3i8itM5cCs=
modernc.org/ccgo/v3 v3.9.4/go.mod h1:19XAY9uOrYnDhOgfHwCABasBvK69jgC4I8+rizbk3Bc=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
modernc.org/libc v1.7.13-0.20210308123627-12f642a52bb8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
modernc.org/libc v1.9.5 h1:zv111ldxmP7DJ5mOIqzRbza7ZDl3kh4ncKfASB2jIYY=
modernc.org/libc v1.9.5/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.2.2 h1:+yFk8hBprV+4c0U9GjFtL+dV3N8hOJ8JCituQcMShFY=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.0.4 h1:utMBrFcpnQDdNsmM6asmyH/FM9TqLPS7XF7otpJmrwM=
modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.10.7 h1:B4ITfAx3HxSxOOKZqKhw4vnrhM+kTY1HoJf2L7PQBCQ=
modernc.org/sqlite v1.10.7/go.mod h1:GXpJIZPNgRGqG0inyYDW18j9YpBpFUBn/weGI63hLLs=
modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/tcl v1.5.2 h1:sYNjGr4zK6cDH74USl8wVJRrvDX6UOLpG0j4lFvR0W0=
modernc.org/tcl v1.5.2/go.mod h1:pmJYOLgpiys3oI4AeAafkcUfE+TKKilminxNyU/+Zlo=
modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA=
modernc.org/z v1.0.1 h1:WyIDpEpAIx4Hel6q/Pcgj/VhaQV5XPJ2I6ryIYbjnpc=
modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA=

D jobs.go => jobs.go +0 -113
@@ 1,113 0,0 @@
package main

import (
	"context"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/gammazero/workerpool"
)

func startDownloadVerify(files map[string]string, lfiles map[string]string) (bool, []error) {
	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)

	wp := workerpool.New(opt.Threads)

	completed := true // will be set to false by the functions below

	// Errors, mutex because workers report their own errors concurrently.
	errMutex := &sync.Mutex{}
	var errors []error

	errc := make(chan bool, opt.Threads) // used to signal an error occured, buffered in case multiple errors occured
	endc := make(chan bool, 1)           // Thread to tell this goroutine to stop checking for signals
	go func() {
		for {
			select {
			case <-sigc:
				log.Println("Signal recieved, gracefully shutting down...")
				completed = false
				wp.Stop()
				return
			case <-errc:
				log.Println("Error occured, gracefully stopping...")
				completed = false
				wp.Stop()
				return
			case <-endc:
				return
			}
		}
	}()

	// We're going to pause so that all jobs will be submitted by the time
	// any errors will occur (when it starts)
	pctx, c := context.WithCancel(context.Background())
	wp.Pause(pctx)

	for k, v := range files {
		var lv *string
		if _, ok := lfiles[k]; ok {
			s := lfiles[k]
			lv = &s
		}

		{ // new scope since we're dealing with a closure

			// set variables within our scope to the outside one.
			k := k
			v := v
			lv := lv

			f := func() {
				err := downloadAndVerify(k, v, lv)
				if err != nil {
					errMutex.Lock()
					defer errMutex.Unlock()
					errors = append(errors, err)
					if len(errors) > 0 {
						completed = false
						errc <- true
						return
					}
				}
			}
			wp.Submit(f)
		} // end scope
	}

	// Unpause
	c()
	// New channel, we'll wait for a signal to go ahead and exit
	// the function
	donec := make(chan bool)

	go func() {
		// hack, I'm going to fork the worker library so that
		// we can wait for a worker pool to stop instead of finish
		// its queue using a function like WaitStop
		for {
			if wp.WaitingQueueSize() == 0 && !wp.Stopped() {
				wp.Stop()
				donec <- true
				return
			}

			if wp.Stopped() {
				donec <- true
				return
			}

			time.Sleep(8 * time.Millisecond)
		}

	}()

	<-donec
	return completed, errors
}

D local.go => local.go +0 -77
@@ 1,77 0,0 @@
package main

// This file is for local storage related stuff. We intentionally
// do not store a local sqlite database for performance and file size reasons.

// All that we need to store is the current revision of our installation, however
// this may change in the future. Storing this as JSON ensures backwards compatability.

// Oh yeah, and a lockfile.

import (
	"encoding/json"
	"log"
	"os"
	"time"
)

type localManifest struct {
	DownloadedRevision int `json:"downloadedRevision"` // the currently installed revision
}

var manifest localManifest

var lmd string // path to local manifest
var lfd string // path to lockfile

func (m *localManifest) write() error {
	if !fileExists(lmd) {
		_, err := os.Create(lmd)
		if err != nil {
			return err
		}
	}

	b, err := json.Marshal(&manifest)
	if err != nil {
		return err
	}

	err = os.WriteFile(lmd, b, 0777)
	if err != nil {
		return err
	}
	return nil
}

func (m *localManifest) read() error {
	b, err := os.ReadFile(lmd)
	if err != nil {
		return err
	}
	err = json.Unmarshal(b, m)
	if err != nil {
		return err
	}

	return nil
}

func lockInstall() error {
	err := os.WriteFile(lfd, []byte(time.Now().String()), 0777)
	return err
}

func unlockInstall() {
	err := os.Remove(lfd)
	if err != nil {
		log.Fatal("Failed to unlock installation: " + err.Error())
	}
}

func isLocked() bool {
	if fileExists(lfd) {
		return true
	}
	return false
}

M main.go => main.go +22 -138
@@ 1,153 1,37 @@
package main

import (
	"crypto/rsa"
	"database/sql"
	"log"
	"os"
	"path/filepath"
)

// If this is set we're doing a first-time installation
var install bool

// The developer public key
var publicKey *rsa.PublicKey

func init() {
	log.SetFlags(0)
}
	"github.com/integrii/flaggy"
)

func main() {
	if fileExists(lfd) && !opt.IgnoreLockfile {
		log.Fatal("Lockfile exists, are you running another installation? Re-run with -b to run regardless of the lockfile.")
	}

	if !fileExists(lmd) {
		install = true
	}

	m, err := get(opt.URL + "/ofmanifest.db")
	if err != nil {
		log.Fatal(err)
	}
	url := "https://toast.openfortress.fun/toast/"
	var dir string
	upgrade := flaggy.NewSubcommand("upgrade")
	upgrade.Description = "Upgrades your game files to the latest version."
	upgrade.String(&url, "u", "url", "url of tvs repository with a slash at the end, defaults to Open Fortress' servers.")
	upgrade.AddPositionalValue(&dir, "directory", 1, true, "directory to interact with")
	flaggy.AttachSubcommand(upgrade, 1)

	// In ofatomic, to ensure the files are legitimate it checks
	// *every file* for if the signature is correct. This is bad for
	// performance. Instead we verify the database and then trust its
	// hashes.
	if !opt.DisableSigning {
		ms, err := get(opt.URL + "ofmanifest.sig")
		if err != nil {
			log.Fatal(err)
		}
		if !verifySig(ms, m) {
			log.Fatal("invalid database signature")
		}
	}
	var tvsdir string
	toast := flaggy.NewSubcommand("toast")
	toast.Description = "Toasts files. Don't use this unless you know what you're doing."
	toast.AddPositionalValue(&dir, "game files", 1, true, "directory of game files to reference")
	toast.AddPositionalValue(&tvsdir, "tvs directory", 2, true, "directory of tvs to add changes to")
	flaggy.AttachSubcommand(toast, 1)

	cd, err := os.UserCacheDir()
	if err != nil {
		log.Fatal("Failed to determine cache directory: " + err.Error())
	}
	flaggy.Parse()

	mcd := filepath.Join(cd, "murse")
	tf := filepath.Join(mcd, "ofmanifest.db")

	err = os.WriteFile(tf, *m, 0777)
	if err != nil {
		log.Fatal(err)
	if upgrade.Used {
		os.Exit(upgradeMain(dir, url))
	}

	db, err := sql.Open("sqlite", tf)
	if err != nil {
		log.Fatal(err)
	}

	defer db.Close()

	var lrev int
	var localFiles map[string]string
	if !install {
		err = manifest.read()
		if err != nil {
			log.Fatal(err)
		}

		localFiles, err = filesFromSQLite(db, manifest.DownloadedRevision)
		if err != nil {
			log.Fatal(err)
		}
		lrev = manifest.DownloadedRevision
	}

	nrev, err := latestRevisionFromSQL(db)
	if err != nil {
		log.Fatal(err)
	}

	newFiles, err := getRevisedFiles(db, lrev, nrev)
	if err != nil {
		log.Fatal(err)
	}

	if !fileExists(opt.Directory) {
		err := os.Mkdir(opt.Directory, 0700)
		if err != nil {
			log.Fatal("could not create game directory: " + err.Error())
		}
	}

	confdir := filepath.Join(opt.Directory, ".murse")

	if !fileExists(confdir) {
		err := os.Mkdir(confdir, 0700)
		if err != nil {
			log.Fatal("unable to create .murse directory: " + err.Error())
		}
	}

	lockInstall()
	defer unlockInstall()

	s, errs := startDownloadVerify(newFiles, localFiles)
	if !s {
		if len(errs) != 0 {
			for _, v := range errs {
				log.Println(v)
			}
			log.Fatal("Operation failed due to errors. Include this log if you file a bug report.")
		} else {
			log.Fatal("Download not completed.")
		}
		unlockInstall()
		os.Exit(1)

	}
	// Sadly ofatomic has no sense of files being removed
	//	if !install {
	//		remOutdated()
	//	}

	manifest.DownloadedRevision = lrev
	err = manifest.write()
	if err != nil {
		log.Fatal("Couldn't write manifest file, any custom files may be overwritten unless it is fixed: " + err.Error())
	}

	// Now we update the gameinfo.txt, so that the game will launch once everything
	// has been downloaded.
	if install {
		b, err := get("https://raw.githubusercontent.com/int-72h/ofatomic/main/ofatomic/gameinfo.txt")
		if err != nil {
			log.Fatal("Unable to download gameinfo.txt, game will not start: " + err.Error())
		}

		err = os.WriteFile(filepath.Join(opt.Directory, "gameinfo.txt"), *b, 0777)
		if err != nil {
			log.Fatal("Failed to write gameinfo.txt to disk: " + err.Error())
		}
	if toast.Used {
		os.Exit(toastMain(dir, tvsdir))
	}

	log.Println("Completed successfully. Happy fragging!")
	flaggy.ShowHelp("Please choose a subcommand.")
	os.Exit(1)
}

D sql.go => sql.go +0 -46
@@ 1,46 0,0 @@
package main

import (
	"database/sql"
	"log"

	_ "modernc.org/sqlite"
)

// Returns the latest revision number from a given SQLite connection
func latestRevisionFromSQL(db *sql.DB) (int, error) {
	rw, err := db.Query("SELECT revision FROM files WHERE revision = (SELECT MAX(revision) FROM files);")
	if err != nil {
		log.Fatal(err)
	}
	defer rw.Close()
	var rev int
	_ = rw.Next()
	err = rw.Scan(&rev)
	if err != nil {
		return 0, err
	}
	return rev, nil
}

// Returns a map of a file name and its hashes from a given revision.
func filesFromSQLite(db *sql.DB, rev int) (map[string]string, error) {
	rws, err := db.Query("SELECT path, checksum FROM files WHERE revision = ?;", rev)
	if err != nil {
		log.Fatal(err)
	}
	defer rws.Close()

	m := make(map[string]string)

	var path string
	var checksum string
	for rws.Next() {
		err := rws.Scan(&path, &checksum)
		if err != nil {
			log.Fatal(err)
		}
		m[path] = checksum
	}
	return m, nil
}

A toaster.go => toaster.go +363 -0
@@ 0,0 1,363 @@
package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"io"
	"io/fs"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strconv"
)

func jFile2Change(bpath string, path string, folder bool, chch chan Change) error {
	relpath, err := filepath.Rel(bpath, path)
	if err != nil {
		return err
	}

	if folder {
		chch <- Change{
			Type: TYPE_MKDIR,
			Path: relpath,
		}

		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		return err
	}

	defer file.Close()

	hasher := md5.New()
	io.Copy(hasher, file)
	hash := hasher.Sum(nil)

	objectid := sfnode.Generate().Base64()

	chch <- Change{
		Type:   TYPE_WRITE,
		Object: objectid,
		Path:   relpath,
		MD5:    hex.EncodeToString(hash),
	}

	return nil
}

func compareAccuChanges(oldcg []Change, newcg []Change) []Change {
	var finalcg []Change

	// Compare the added/changed files
	// Convert array to map for easier/quicker look-ups.

	// In testing adding concurrency to this part made
	// it about five times slower (10MS -> 50MS). This
	// was tested going from an empty folder to a full
	// Open Fortress installation.

	// Convert to a map for faster lookups and
	// simpler code.
	oldMap := rev2Map(oldcg)
	for _, v := range newcg {
		chg, ok := oldMap[v.Path]
		if !ok {
			finalcg = append(finalcg, v)
		} else if v.Type == TYPE_WRITE && v.MD5 != chg.MD5 {
			finalcg = append(finalcg, v)
		}
	}

	// Next we compare files and directories not
	// in the newer version.

	newMap := rev2Map(newcg)
	for _, v := range oldcg {
		if _, ok := newMap[v.Path]; !ok {
			finalcg = append(finalcg, invertChange(v))
		}
	}

	return finalcg
}

// For performance reasons in algorithms a map is
// much faster than iterating over the entire array.
func rev2Map(changes []Change) map[string]Change {
	chmap := make(map[string]Change, len(changes))
	for _, v := range changes {
		chmap[v.Path] = v
	}

	return chmap
}

/*
func map2Rev(chmap map[string]Change) []Change {
	charray := make([]Change, 0)
	for _, v := range chmap {
		charray = append(charray, v)
	}

	return charray
}
*/

func invertChange(cg Change) Change {
	switch {
	case cg.Type == TYPE_WRITE || cg.Type == TYPE_MKDIR:
		cg.Type = TYPE_DEL
		return cg
	case cg.Type == TYPE_DEL:
		cg.Type = TYPE_WRITE
		return cg
	}

	return cg
}

type cumlcache struct {
	Version int      `json:"version"`
	Changes []Change `json:"changes"`
}

func generateCumlCache(dir string) error {
	errPrintln("Generating the cumulative cache...")
	var revisions [][]Change
	i := -1
	for ; ; i++ {
		file, err := os.Open(filepath.Join(dir, "revisions", strconv.Itoa(i+1)))
		if err != nil {
			if os.IsNotExist(err) {
				break
			}

			debug.PrintStack()
			return err
		}
		defer file.Close()

		var change []Change

		err = json.NewDecoder(file).Decode(&change)
		if err != nil {
			debug.PrintStack()
			return err
		}
		revisions = append(revisions, change)
	}

	cuml := cumlcache{
		Version: i,
		Changes: replayChanges(revisions),
	}
	bytes, err := json.Marshal(cuml)
	if err != nil {
		debug.PrintStack()
		return err
	}

	err = os.WriteFile(filepath.Join(dir, "cumlcache"), bytes, 0777)
	if err != nil {
		debug.PrintStack()
	}

	return err

}

func readCumlCache(dir string) (cumlcache, error) {
	file, err := os.Open(filepath.Join(dir, "cumlcache"))
	if err != nil {
		if !os.IsNotExist(err) {
			debug.PrintStack()
			return cumlcache{}, err
		}

		err = generateCumlCache(dir)
		if err != nil {
			debug.PrintStack()
			return cumlcache{}, err
		}

		file, err = os.Open(filepath.Join(dir, "cumlcache"))
		if err != nil {
			return cumlcache{}, err
		}
	}

	defer file.Close()

	var cuml cumlcache
	err = json.NewDecoder(file).Decode(&cuml)
	if err != nil {
		debug.PrintStack()
	}

	return cuml, err
}

func dir2Change(bpath string) ([]Change, []error) {
	pool := NewPool(runtime.NumCPU())

	var changes []Change
	chch := make(chan Change)

	var count int

	walkfunc := func(path string, info fs.FileInfo, err error) error {
		if err != nil || path == bpath {
			return err
		}

		count++

		go func() {
			pool.Jobch <- func() error { return jFile2Change(bpath, path, info.IsDir(), chch) }
		}()

		return nil
	}

	err := filepath.Walk(bpath, walkfunc)
	if err != nil {
		log.Fatal(err)
	}

	var errs []error
	for {
		select {
		case change := <-chch:
			changes = append(changes, change)
			if len(changes) == count {
				pool.Stop()
				goto postloop
			}
		case err := <-pool.Errch:
			errs = append(errs, err)
			pool.Stop()
			for {
				select {
				case err := <-pool.Errch:
					errs = append(errs, err)
				default:
					goto postloop
				}
			}
		}
	}

postloop:
	return changes, errs

}

func toastMain(filesdir string, tvsdir string) int {
	err := os.MkdirAll(filepath.Join(tvsdir, "objects"), 0777)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	err = os.MkdirAll(filepath.Join(tvsdir, "revisions"), 0777)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	cuml, err := readCumlCache(tvsdir)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	errPrintln("Reading filesystem...")
	fscuml, errs := dir2Change(filepath.Base(filesdir))
	if len(errs) != 0 {
		for _, err := range errs {
			errPrintln(err)
		}
		return 1
	}

	errPrintln("Comparing changes...")
	revision := compareAccuChanges(cuml.Changes, fscuml)
	if len(revision) == 0 {
		errPrintln("No changes were found.")
		return 1
	}

	errPrintln("Copying objects...")
	for _, v := range revision {
		if v.Type == TYPE_WRITE {
			old, err := os.Open(filepath.Join(filesdir, v.Path))
			if err != nil {
				debug.PrintStack()
				errPrintln(err)
				return 1
			}
			defer old.Close()

			new, err := os.OpenFile(filepath.Join(tvsdir, "objects", v.Object), os.O_CREATE|os.O_WRONLY, 0777)
			if err != nil {
				debug.PrintStack()
				errPrintln(err)
				return 1
			}
			defer new.Close()

			_, err = io.Copy(new, old)
			if err != nil {
				debug.PrintStack()
				errPrintln(err)
				return 1
			}

			old.Close()
			new.Close()
		}
	}

	errPrintln("Writing revision...")
	b, err := json.Marshal(revision)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	f, err := os.OpenFile(filepath.Join(tvsdir, "revisions", strconv.Itoa(cuml.Version+1)), os.O_CREATE|os.O_WRONLY, 0777)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	defer f.Close()

	_, err = f.Write(b)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	err = generateCumlCache(tvsdir)
	if err != nil {
		debug.PrintStack()
		errPrintln(err)
		return 1
	}

	return 0
}

A upgrade.go => upgrade.go +141 -0
@@ 0,0 1,141 @@
package main

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime/debug"
	"strconv"
)

func upgradeMain(dir string, url string) int {
	instRev, err := getInstalledRevision(dir)
	if err != nil {
		errPrintln(err)
		return 1
	}

	client := NewClient(url)

	latestRev, err := client.GetLatestRevision()
	if err != nil {
		errPrintln(err)
		return 1
	}

	errPrintln(instRev, "->", latestRev)

	var revisions [][]Change
	for i := instRev + 1; i != latestRev+1; i++ {
		rev, err := client.GetRevision(i)
		if err != nil {
			debug.PrintStack()
			errPrintln(err)
			return 1
		}

		revisions = append(revisions, rev)
	}

	changes := replayChanges(revisions)
	writes, mkdirs, dels := splitChanges(changes)
	err = os.RemoveAll(filepath.Join(dir, ".revision"))
	if err != nil {
		errPrintln(err)
		return 1
	}

	for _, v := range dels {
		fmt.Println("DEL", v.Path)
		path := filepath.FromSlash(v.Path)
		err := os.Remove(filepath.Join(dir, path))
		if err != nil && !os.IsNotExist(err) {
			errPrintln(err)
			return 1
		}
	}

	for _, v := range mkdirs {
		path := filepath.FromSlash(v.Path)
		err := os.MkdirAll(filepath.Join(dir, path), 0777)
		if err != nil {
			errPrintln(err)
			return 1
		}
	}

	pool := NewPool(16)
	jobs := len(writes)
	done := 0

	for _, v := range writes {
		change := v
		f := func() error {
			fmt.Println("WRITE", change.Path)
			object, err := client.GetObject(change.Object)
			if err != nil {
				return err
			}

			defer object.Close()

			path := filepath.FromSlash(change.Path)
			file, err := os.Create(filepath.Join(dir, path))
			if err != nil {
				return err
			}

			defer file.Close()

			_, err = io.Copy(file, object)
			return err
		}

		pool.Jobch <- f

		select {
		case err := <-pool.Errch:
			if err != nil {
				pool.Stop()
				errPrintln(err)
				for {
					select {
					case err := <-pool.Errch:
						errPrintln(err)
					default:
						goto postloop
					}
				}
			postloop:
				return 1
			}

			done++
		default:
			continue
		}

		if done >= jobs {
			pool.Stop()
			break
		}
	}

	file, err := os.Create(filepath.Join(dir, ".revision"))
	if err != nil {
		errPrintln(err)
		return 1
	}

	defer file.Close()

	_, err = file.Write([]byte(strconv.Itoa(latestRev)))
	if err != nil {
		errPrintln(err)
		return 1
	}

	return 0

}

A util.go => util.go +14 -0
@@ 0,0 1,14 @@
package main

import (
	"fmt"
	"os"

	"github.com/bwmarrin/snowflake"
)

var sfnode, _ = snowflake.NewNode(0)

func errPrintln(a ...any) {
	fmt.Fprintln(os.Stderr, a...)
}

A versioning.go => versioning.go +69 -0
@@ 0,0 1,69 @@
package main

import (
	"encoding/json"
)

const (
	TYPE_WRITE = iota
	TYPE_MKDIR
	TYPE_DEL
)

type Change struct {
	Type   uint   `json:"type"`
	Path   string `json:"path"`
	MD5    string `json:"md5"`
	Object string `json:"object"`
}

func (c *Change) MarshalJSON() ([]byte, error) {
	if c.Type == TYPE_WRITE {
		return json.Marshal(map[string]interface{}{
			"type":   c.Type,
			"path":   c.Path,
			"md5":    c.MD5,
			"object": c.Object,
		})
	}

	return json.Marshal(map[string]interface{}{
		"type": c.Type,
		"path": c.Path,
	})
}

func splitChanges(changes []Change) (writes []Change, mkdirs []Change, dels []Change) {
	for _, v := range changes {
		switch v.Type {
		case TYPE_WRITE:
			writes = append(writes, v)
		case TYPE_MKDIR:
			mkdirs = append(mkdirs, v)
		case TYPE_DEL:
			dels = append(dels, v)
		}
	}

	return
}

func mapToChanges(changeMap map[string]Change) []Change {
	var changes []Change
	for _, v := range changeMap {
		changes = append(changes, v)
	}
	return changes
}

func replayChanges(revisions [][]Change) []Change {
	chgMap := make(map[string]Change)

	for _, revision := range revisions {
		for _, change := range revision {
			chgMap[change.Path] = change
		}
	}

	return mapToChanges(chgMap)
}