~fgaz/builds.sr.ht

1c7fd5355178585f6e74ed6d69e3ac0eb3b310b6 — Adnan Maolood 1 year, 6 months ago fcb43a9
worker: Run go fmt and go mod tidy
6 files changed, 55 insertions(+), 61 deletions(-)

M api/go.sum
M worker/context.go
M worker/main.go
M worker/manifest.go
M worker/tasks.go
M worker/triggers.go
M api/go.sum => api/go.sum +0 -2
@@ 31,8 31,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.sr.ht/~sircmpwn/core-go v0.0.0-20220321082727-3f80f677f56d h1:0weIXd6ya99UAY0ZCOIKRIuFCarlxm3d8mI9thx4qHI=
git.sr.ht/~sircmpwn/core-go v0.0.0-20220321082727-3f80f677f56d/go.mod h1:uUqzeO5OLl/nRZfPk0igIAweRZiVwUmu/OGYfjS9fWc=
git.sr.ht/~sircmpwn/core-go v0.0.0-20220530120843-d0bf1153ada4 h1:pyGHngvO7qIANeI5pmqzfVc+nLKSTTuwq2OA66nxEGw=
git.sr.ht/~sircmpwn/core-go v0.0.0-20220530120843-d0bf1153ada4/go.mod h1:uUqzeO5OLl/nRZfPk0igIAweRZiVwUmu/OGYfjS9fWc=
git.sr.ht/~sircmpwn/dowork v0.0.0-20210820133136-d3970e97def3 h1:9WCv5cK67s2SiY/R4DWT/OchEsFnfYDz3lbevKxZ4QI=

M worker/context.go => worker/context.go +19 -19
@@ 267,25 267,25 @@ func (ctx *JobContext) Control(
func (ctx *JobContext) SSH(args ...string) *exec.Cmd {
	sport := strconv.Itoa(ctx.Port)
	switch ctx.ImageConfig.LoginCmd {
		case "drawterm":
			return exec.CommandContext(ctx.Context,
				"env", fmt.Sprintf("DIALSRV=%s", sport),
				"PASS=password", "drawterm",
				"-a", "none",
				"-u", "glenda",
				"-h", "127.0.0.1",
				"-Gc", strings.Join(args, " "))
		case "ssh":
			return exec.CommandContext(ctx.Context, "ssh",
				append([]string{"-q",
					"-p", sport,
					"-o", "UserKnownHostsFile=/dev/null",
					"-o", "StrictHostKeyChecking=no",
					"-o", "LogLevel=quiet",
					"build@localhost",
				}, args...)...)
		default:
			panic(errors.New("Unknown login command"))
	case "drawterm":
		return exec.CommandContext(ctx.Context,
			"env", fmt.Sprintf("DIALSRV=%s", sport),
			"PASS=password", "drawterm",
			"-a", "none",
			"-u", "glenda",
			"-h", "127.0.0.1",
			"-Gc", strings.Join(args, " "))
	case "ssh":
		return exec.CommandContext(ctx.Context, "ssh",
			append([]string{"-q",
				"-p", sport,
				"-o", "UserKnownHostsFile=/dev/null",
				"-o", "StrictHostKeyChecking=no",
				"-o", "LogLevel=quiet",
				"build@localhost",
			}, args...)...)
	default:
		panic(errors.New("Unknown login command"))
	}
}


M worker/main.go => worker/main.go +2 -2
@@ 10,12 10,12 @@ import (
	"runtime"
	"sync"

	"git.sr.ht/~sircmpwn/core-go/crypto"
	goredis "github.com/go-redis/redis/v8"
	"github.com/vaughan0/go-ini"
	"git.sr.ht/~sircmpwn/core-go/crypto"

	_ "github.com/lib/pq"
	celery "github.com/gocelery/gocelery"
	_ "github.com/lib/pq"
)

var (

M worker/manifest.go => worker/manifest.go +1 -1
@@ 36,7 36,7 @@ func LoadImageConfig(image string) *ImageConfig {
		LoginCmd:   "ssh",
		GitVariant: "git",
		Homedir:    "/home/build",
		Preamble:   `#!/usr/bin/env bash
		Preamble: `#!/usr/bin/env bash
. ~/.buildenv
set -xe


M worker/tasks.go => worker/tasks.go +10 -14
@@ 151,7 151,7 @@ func (ctx *JobContext) SendTasks() error {
			break
		}
		taskpath := path.Join(taskdir, name)
		script = ctx.ImageConfig.Preamble+script+"\n"
		script = ctx.ImageConfig.Preamble + script + "\n"
		if err := ctx.Tee(taskpath, []byte(script)); err != nil {
			return err
		}


@@ 399,7 399,7 @@ func (ctx *JobContext) ConfigureRepos() error {
}

func (ctx *JobContext) CloneGitRepo(srcurl, repo_name, ref string) error {
	git := ctx.SSH("GIT_SSH_COMMAND='ssh -o " +
	git := ctx.SSH("GIT_SSH_COMMAND='ssh -o "+
		"UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'",
		"git", "clone", srcurl, repo_name)
	git.Stdout = ctx.LogFile


@@ 420,7 420,7 @@ func (ctx *JobContext) CloneGitRepo(srcurl, repo_name, ref string) error {
			return errors.Wrap(err, "git checkout")
		}
	}
	git = ctx.SSH("GIT_SSH_COMMAND='ssh -o " +
	git = ctx.SSH("GIT_SSH_COMMAND='ssh -o "+
		"UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'",
		"sh", "-euxc",
		fmt.Sprintf("'cd %s && git submodule update --init --recursive'", repo_name))


@@ 507,8 507,8 @@ func (ctx *JobContext) CloneRepos() error {
			}
		case "hg":
			hg := ctx.SSH("hg", "clone",
				"-e", "'ssh -o UserKnownHostsFile=/dev/null " +
				"-o StrictHostKeyChecking=no'", srcurl, repo_name)
				"-e", "'ssh -o UserKnownHostsFile=/dev/null "+
					"-o StrictHostKeyChecking=no'", srcurl, repo_name)
			hg.Stdout = ctx.LogFile
			hg.Stderr = ctx.LogFile
			if err := hg.Run(); err != nil {


@@ 639,20 639,16 @@ func (ctx *JobContext) UploadArtifacts() error {
	)
	err := errors.New("Build artifacts were requested, but S3 " +
		"is not configured for this build runner.")
	if upstream, ok = config.Get("objects", "s3-upstream");
		!ok || upstream == "" {
	if upstream, ok = config.Get("objects", "s3-upstream"); !ok || upstream == "" {
		return err
	}
	if accessKey, ok = config.Get("objects", "s3-access-key");
		!ok || accessKey == "" {
	if accessKey, ok = config.Get("objects", "s3-access-key"); !ok || accessKey == "" {
		return err
	}
	if secretKey, ok = config.Get("objects", "s3-secret-key");
		!ok || secretKey == "" {
	if secretKey, ok = config.Get("objects", "s3-secret-key"); !ok || secretKey == "" {
		return err
	}
	if bucket, ok = config.Get("builds.sr.ht::worker", "s3-bucket");
		!ok || bucket == "" {
	if bucket, ok = config.Get("builds.sr.ht::worker", "s3-bucket"); !ok || bucket == "" {
		return err
	}
	if prefix, ok = config.Get("builds.sr.ht::worker", "s3-prefix"); !ok {


@@ 676,7 672,7 @@ func (ctx *JobContext) UploadArtifacts() error {
	}
	for _, src := range ctx.Manifest.Artifacts {
		ctx.Log.Printf("Uploading %s", src)
		name := path.Join(prefix, "~" + ctx.Job.Username,
		name := path.Join(prefix, "~"+ctx.Job.Username,
			strconv.Itoa(ctx.Job.Id),
			hex.EncodeToString(random),
			filepath.Base(src))

M worker/triggers.go => worker/triggers.go +23 -23
@@ 18,12 18,12 @@ import (
	"time"
	"unicode/utf8"

	"git.sr.ht/~sircmpwn/core-go/crypto"
	"github.com/martinlindhe/base36"
	ms "github.com/mitchellh/mapstructure"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"git.sr.ht/~sircmpwn/core-go/crypto"
	gomail "gopkg.in/mail.v2"
	ms "github.com/mitchellh/mapstructure"
)

var (


@@ 64,10 64,10 @@ type JobStatus struct {
}

type JobGroupStatus struct {
	Id       int          `json:"id"`
	Note     *string      `json:"note"`
	Owner    JobOwner     `json:"owner"`
	Jobs     []JobStatus  `json:"jobs"`
	Id    int         `json:"id"`
	Note  *string     `json:"note"`
	Owner JobOwner    `json:"owner"`
	Jobs  []JobStatus `json:"jobs"`
}

type EmailTrigger struct {


@@ 98,8 98,8 @@ func (ctx *JobContext) ProcessTriggers() {
		var trigger Trigger
		ms.Decode(def, &trigger)
		failures := map[string]interface{}{
			"failed": nil,
			"timeout": nil,
			"failed":    nil,
			"timeout":   nil,
			"cancelled": nil,
		}
		process := trigger.Condition == "always"


@@ 211,7 211,7 @@ func (ctx *JobContext) processEmail(def map[string]interface{}) {
		Tasks    string
	}
	tmpl, err := template.New("email").Parse(
`{{if .Job.Tags}}{{.Job.Tags}}{{else}}Job{{end}} #{{.Job.Id}}: {{.Status}} in {{.Duration}}
		`{{if .Job.Tags}}{{.Job.Tags}}{{else}}Job{{end}} #{{.Job.Id}}: {{.Status}} in {{.Duration}}

{{if .Job.Note}}{{.Job.Note}}



@@ 253,9 253,9 @@ func (ctx *JobContext) processWebhook(def map[string]interface{}) {
			*ctx.Job.Runner, ctx.Job.Id),
		Note:   ctx.Job.Note,
		Runner: ctx.Job.Runner,
		Owner: JobOwner {
		Owner: JobOwner{
			CName: "~" + ctx.Job.Username,
			Name: ctx.Job.Username,
			Name:  ctx.Job.Username,
		},
	}



@@ 270,7 270,7 @@ func (ctx *JobContext) processWebhook(def map[string]interface{}) {
			return
		}
		task := TaskStatus{
			Name: name,
			Name:   name,
			Status: taskStatus,
			Log: fmt.Sprintf("http://%s/logs/%d/%s/log",
				*ctx.Job.Runner, ctx.Job.Id, name),


@@ 293,7 293,7 @@ func (ctx *JobContext) processWebhook(def map[string]interface{}) {
	nonce, sig := crypto.SignWebhook(data)

	ctx.Log.Println("Sending webhook...")
	client := &http.Client{Timeout: time.Second*10}
	client := &http.Client{Timeout: time.Second * 10}
	req, err := http.NewRequest("POST", trigger.Url, bytes.NewReader(data))
	req.Header.Add("X-Payload-Nonce", nonce)
	req.Header.Add("X-Payload-Signature", sig)


@@ 362,8 362,8 @@ func (ctx *JobContext) processJobGroupTriggers(groupId int) {
			return
		}
		failures := map[string]interface{}{
			"failed": nil,
			"timeout": nil,
			"failed":    nil,
			"timeout":   nil,
			"cancelled": nil,
		}
		process := trigger.Condition == "always"


@@ 374,7 374,7 @@ func (ctx *JobContext) processJobGroupTriggers(groupId int) {
			process = process || trigger.Condition == "success"
		}
		triggers := map[string]func(*JobGroup, string, map[string]interface{}){
			"email":   ctx.processGroupEmail,
			"email": ctx.processGroupEmail,
			//"webhook": ctx.processGroupWebhook, TODO
		}
		if process {


@@ 487,14 487,14 @@ func (ctx *JobContext) processGroupEmail(group *JobGroup, status string,
		origin := ctx.Conf("builds.sr.ht", "origin")
		url := fmt.Sprintf("%s/~%s/job/%d", origin, job.Username, job.Id)
		if job.Tags != nil {
			jobsBuf.WriteString(fmt.Sprintf("%c #%d %-" +
				strconv.FormatInt(int64(maxStatus), 10) + "s %-" +
				strconv.FormatInt(int64(maxTags), 10) + "s %s\n",
			jobsBuf.WriteString(fmt.Sprintf("%c #%d %-"+
				strconv.FormatInt(int64(maxStatus), 10)+"s %-"+
				strconv.FormatInt(int64(maxTags), 10)+"s %s\n",
				statusChar, job.Id, strings.ToUpper(job.Status), *job.Tags, url))
		} else {
			jobsBuf.WriteString(fmt.Sprintf("%c #%d %-" +
				strconv.FormatInt(int64(maxStatus), 10) + "s %-" +
				strconv.FormatInt(int64(maxTags), 10) + "s %s\n",
			jobsBuf.WriteString(fmt.Sprintf("%c #%d %-"+
				strconv.FormatInt(int64(maxStatus), 10)+"s %-"+
				strconv.FormatInt(int64(maxTags), 10)+"s %s\n",
				statusChar, job.Id, strings.ToUpper(job.Status), job.Image, url))
		}
	}


@@ 508,7 508,7 @@ func (ctx *JobContext) processGroupEmail(group *JobGroup, status string,
	}
	// TODO: Use job group tags here
	tmpl, err := template.New("group-email").Parse(
`{{if .Tag}}{{.Tag}}{{else}}Job group{{end}}: {{.Status}} in {{.Duration}}
		`{{if .Tag}}{{.Tag}}{{else}}Job group{{end}}: {{.Status}} in {{.Duration}}

{{if .Group.Note}}{{.Group.Note}}{{end}}