~evanj/cms

977e0de28ca8f9d30d073665642e89fddfde493f — Evan J 2 months ago 356e3f3
Feat(cms.go): Retrying if requests last too long. Effective max request
time of 30 seconds (ten seconds max retried three times).
4 files changed, 91 insertions(+), 23 deletions(-)

M .build.yml
M cms.go
M internal/c/c.go
M makefile
M .build.yml => .build.yml +10 -10
@@ 14,18 14,18 @@ tasks:
      make setup gen build
      if [ `git branch | head -n 1 | awk '{print$5}' | sed 's/)//g' | xargs -I X bash -c "git log --pretty=oneline origin/master | head -n 1 | grep X" | wc -l` == "1" ]; then
        echo "deploying to production"
        ssh -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo mysqldump cms > \$(mktemp --suffix .cms.datbase.backup.sql)"'
        ssh -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "rm ~/cms/cms 2>/dev/null"'
        scp -o 'StrictHostKeyChecking=no' cms evan@140.82.14.80:~/cms
        scp -r -o 'StrictHostKeyChecking=no' static evan@140.82.14.80:~/cms
        ssh -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo systemctl restart cms"'
        ssh -p 4545 -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo mysqldump cms > \$(mktemp --suffix .cms.datbase.backup.sql)"'
        ssh -p 4545 -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "rm ~/cms/cms 2>/dev/null"'
        scp -p 4545 -o 'StrictHostKeyChecking=no' cms evan@140.82.14.80:~/cms
        scp -p 4545 -r -o 'StrictHostKeyChecking=no' static evan@140.82.14.80:~/cms
        ssh -p 4545 -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo systemctl restart cms"'
      elif [ `git branch | head -n 1 | awk '{print$5}' | sed 's/)//g' | xargs -I X bash -c "git log --pretty=oneline origin/tip | head -n 1 | grep X" | wc -l` == "1" ]; then
        echo "deploying to tip"
        ssh -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo mysqldump cms_tip > \$(mktemp --suffix .cms.tip.datbase.backup.sql)"'
        ssh -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "rm ~/cms.tip/cms 2>/dev/null"'
        scp -o 'StrictHostKeyChecking=no' cms evan@140.82.14.80:~/cms.tip
        scp -r -o 'StrictHostKeyChecking=no' static evan@140.82.14.80:~/cms.tip
        ssh -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo systemctl restart cms.tip"'
        ssh -p 4545 -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo mysqldump cms_tip > \$(mktemp --suffix .cms.tip.datbase.backup.sql)"'
        ssh -p 4545 -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "rm ~/cms.tip/cms 2>/dev/null"'
        scp -p 4545 -o 'StrictHostKeyChecking=no' cms evan@140.82.14.80:~/cms.tip
        scp -p 4545 -r -o 'StrictHostKeyChecking=no' static evan@140.82.14.80:~/cms.tip
        ssh -p 4545 -o 'StrictHostKeyChecking=no' evan@140.82.14.80 'bash -c "sudo systemctl restart cms.tip"'
      else 
        echo "not deploying"
      fi

M cms.go => cms.go +65 -3
@@ 2,6 2,7 @@
package main

import (
	"bytes"
	"context"
	"log"
	"net/http"


@@ 35,8 36,69 @@ func (a *App) ServeHTTP(w http.ResponseWriter, r *http.Request) {
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	a.handleWithRetry(w, r, h)
}

type wrappedResponseWriter struct {
	w  http.ResponseWriter
	h  http.Header
	sc int
	b  bytes.Buffer
}

func (rw *wrappedResponseWriter) Header() http.Header             { return rw.h }
func (rw *wrappedResponseWriter) Write(bytes []byte) (int, error) { return rw.b.Write(bytes) }
func (rw *wrappedResponseWriter) WriteHeader(statusCode int)      { rw.sc = statusCode }

func (rw *wrappedResponseWriter) Flush() error {
	rw.w.WriteHeader(rw.sc)
	_, err := rw.w.Write(rw.b.Bytes())
	return err
}

// handleWithRetry will retry requests when we're approaching long request
// times in an effort to lower tail latency.
func (a *App) handleWithRetry(w http.ResponseWriter, r *http.Request, h http.Handler) {
	var (
		// Effective max request time is 30 seconds (ten seconds will be retried
		// three times).
		maxAttempts = 3
		maxTime     = 10 * time.Second
		err         error
	)

	for i := 0; i < maxAttempts; i++ {
		var (
			ctx, cancel = context.WithTimeout(context.Background(), maxTime)
			wWrap       = &wrappedResponseWriter{w: w, h: http.Header{}, b: bytes.Buffer{}}
			successCh   = handle(wWrap, r.WithContext(ctx), h)
		)
		defer cancel()

		select {
		case <-ctx.Done():
			err = ctx.Err()
		case <-successCh:
			if err := wWrap.Flush(); err != nil {
				// TODO: Is there a more appropriate way to handle an error here?...
				a.log.Println(err)
			}
			return
		}
	}

	// TODO: Probably want a better error message.
	w.WriteHeader(http.StatusInternalServerError)
	w.Write([]byte(err.Error()))
}

func handle(w http.ResponseWriter, r *http.Request, h http.Handler) chan struct{} {
	c := make(chan struct{})

	go func(c chan struct{}) {
		h.ServeHTTP(w, r)
		c <- struct{}{}
	}(c)

	h.ServeHTTP(w, r.WithContext(ctx))
	return c
}

M internal/c/c.go => internal/c/c.go +4 -1
@@ 159,7 159,10 @@ func (c *Controller) HTML(w http.ResponseWriter, r *http.Request, tmpl *template
	w.Header().Add("Pragma", "no-cache")
	w.Header().Add("Expires", "Sat, 26 Jul 1997 05:00:00 GMT")
	w.WriteHeader(http.StatusOK)
	io.Copy(w, &buf)
	if _, err := io.Copy(w, &buf); err != nil {
		// TODO: Is there a more appropriate way to handle an error here?...
		c.log.Println(err)
	}
}

// TODO: You know why this is bad, change it.

M makefile => makefile +12 -9
@@ 1,31 1,34 @@
BIN=cms
ENV=`cat .env`
VER=`git rev-parse HEAD`
CC=go1.15

all: setup vendor gen build

setup:
	@go get git.sr.ht/~evanj/embed/cmd/embed
	@go get github.com/golang/mock/mockgen
	@go get golang.org/dl/go1.15
	@go1.15 download
	@$(CC) get git.sr.ht/~evanj/embed/cmd/embed
	@$(CC) get github.com/golang/mock/mockgen

vendor: go.mod go.sum
	@go mod tidy
	@go mod vendor
	@$(CC) mod tidy
	@$(CC) mod vendor

build:
	@echo $(VER) | xargs -I {} go build -ldflags='-s -w -X main.build={}' -o $(BIN)
	@echo $(VER) | xargs -I {} $(CC) build -ldflags='-s -w -X main.build={}' -o $(BIN)

gen: 
	@go generate ./...
	@$(CC) generate ./...

test: 
	@env $(ENV) go test ./... -count 1
	@env $(ENV) $(CC) test ./... -count 1

coverage: 
	@env $(ENV) go test ./... -cover -coverprofile=coverage.out ; go tool cover -html=coverage.out
	@env $(ENV) $(CC) test ./... -cover -coverprofile=coverage.out ; $(CC) tool cover -html=coverage.out

lint: 
	@find * -not -name '*_embed.go' | grep -E '*.(sql|go|js|css|html)' | entr -cr go vet ./...
	@find * -not -name '*_embed.go' | grep -E '*.(sql|go|js|css|html)' | entr -cr $(CC) vet ./...

run: gen build
	@clear