~evanj/evanjon.es

d296b66aafe40baf661fba0d6059119badcde74d — Evan M Jones 1 year, 4 months ago 0a9e51e
Feat(cache/async): Improve perf while interacting with CMS.
M evanjon.es.go => evanjon.es.go +13 -20
@@ 12,8 12,8 @@ import (
	"git.sr.ht/~evanj/evanjon.es/internal/c/list"
	"git.sr.ht/~evanj/evanjon.es/internal/c/search"

	// "git.sr.ht/~evanj/evanjon.es/internal/s/cms"
	"git.sr.ht/~evanj/evanjon.es/pkg/cms"
	"git.sr.ht/~evanj/evanjon.es/internal/s/cms"
	api "git.sr.ht/~evanj/evanjon.es/pkg/cms"
)

//go:generate go get git.sr.ht/~evanj/embed


@@ 30,7 30,6 @@ type App struct {

func New(out io.Writer) *App {
	l := log.New(out, "[evanjon.es] ", 0)
	e := c.New(log.New(out, "[evanjon.es:baseRouter] ", 0))

	typePage, err1 := strconv.Atoi(os.Getenv("CMS_TYPE_PAGE"))
	typePost, err2 := strconv.Atoi(os.Getenv("CMS_TYPE_POST"))


@@ 44,25 43,20 @@ func New(out io.Writer) *App {
		l.Fatal("invalid space id")
	}

	// // Transformer, match our data types.
	// cms := cms.New(
	// 	log.New(out, "[evanjon.es:cms] ", 0),
	// 	// API impl.
	// 	api.New(
	// 		os.Getenv("CMS_USER"),
	// 		os.Getenv("CMS_PASS"),
	// 		os.Getenv("CMS_URL"),
	// 		space,
	// 	),
	// 	os.Getenv("CACHE_SECRET"),
	// )
	// Transformer, match our data types.
	cms := cms.New(
		os.Getenv("CMS_USER"),
		os.Getenv("CMS_PASS"),
		os.Getenv("CMS_URL"),
		space,
		log.New(out, "[evanjon.es:cms] ", 0),
		api.New(
			os.Getenv("CMS_USER"),
			os.Getenv("CMS_PASS"),
			os.Getenv("CMS_URL"),
			space,
		),
		os.Getenv("CACHE_SECRET"),
	)

	e := c.New(log.New(out, "[evanjon.es:baseRouter] ", 0), cms)

	return &App{
		log:        l,
		baseRouter: e,


@@ 70,7 64,6 @@ func New(out io.Writer) *App {
		searchRouter: search.New(
			e,
			log.New(out, "[evanjon.es:search] ", 0),
			cms,
			typePage,
			typePost,
			typeMeta,

M internal/c/c.go => internal/c/c.go +18 -7
@@ 7,14 7,21 @@ import (
	"log"
	"net/http"
	"time"

	"git.sr.ht/~evanj/evanjon.es/pkg/cms"
)

type Endpoint struct {
	log *log.Logger
	cms cmsProvider
}

type cmsProvider interface {
	Find(ctx context.Context, typeID int, field, query string) (*cms.Content, error)
}

func New(l *log.Logger) *Endpoint {
	return &Endpoint{l}
func New(l *log.Logger, cms cmsProvider) *Endpoint {
	return &Endpoint{l, cms}
}

func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {


@@ 27,11 34,6 @@ func (e *Endpoint) ContextTimeout(r *http.Request) context.Context {
	return ctx
}

func (e *Endpoint) TODO(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusNotImplemented)
	w.Write([]byte("this website has not yet been completed"))
}

func (e *Endpoint) HTML(w http.ResponseWriter, r *http.Request, t *template.Template, data interface{}) {
	buf := bytes.Buffer{}
	if err := t.Execute(&buf, data); err != nil {


@@ 45,3 47,12 @@ func (e *Endpoint) HTML(w http.ResponseWriter, r *http.Request, t *template.Temp
	w.Header().Add("Content-Type", "text/html")
	w.Write(buf.Bytes())
}

func (e *Endpoint) Find(ctx context.Context, typ int, field, query string, cch chan *cms.Content, ech chan error) {
	c, err := e.cms.Find(ctx, typ, field, query)
	if err != nil {
		ech <- err
		return
	}
	cch <- c
}

M internal/c/list/list.go => internal/c/list/list.go +46 -14
@@ 21,7 21,6 @@ type ListEndpoint struct {

type cmsProvider interface {
	List(ctx context.Context, typeID int, order, field string) ([]cms.Content, error)
	Find(ctx context.Context, typeID int, field, query string) (*cms.Content, error)
}

func New(base *c.Endpoint, log *log.Logger, cms cmsProvider, typePage, typePost, typeMeta int) *ListEndpoint {


@@ 33,27 32,60 @@ func New(base *c.Endpoint, log *log.Logger, cms cmsProvider, typePage, typePost,
	}
}

func (e *ListEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := e.ContextTimeout(r)
	e.log.Println("searching for list with type of:", e.typePost)
func (e *ListEndpoint) list(ctx context.Context) ([]cms.Content, *cms.Content, error) {
	lch := make(chan []cms.Content, 1)
	ach := make(chan *cms.Content, 1)
	ech := make(chan error, 2)

	l, err := e.cms.List(ctx, e.typePost, "desc", "date")
	if err != nil {
		e.log.Println("failed to find list", err)
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("failed to find list"))
		return
	go func(ctx context.Context, typ int, lch chan []cms.Content, ech chan error) {
		l, err := e.cms.List(ctx, typ, "desc", "date")
		if err != nil {
			ech <- err
			return
		}
		lch <- l
	}(ctx, e.typePost, lch, ech)

	go e.Find(ctx, e.typeMeta, "slug", "aside", ach, ech)

	var (
		list  []cms.Content
		aside *cms.Content
	)

	for {
		select {

		case err := <-ech:
			return nil, nil, err
			break

		case l := <-lch:
			list = l
			if aside != nil {
				return list, aside, nil
			}
			break

		case a := <-ach:
			aside = a
			if list != nil {
				return list, aside, nil
			}
			break
		}
	}

	a, err := e.cms.Find(ctx, e.typeMeta, "slug", "aside")
}

func (e *ListEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	l, a, err := e.list(e.ContextTimeout(r))
	if err != nil {
		e.log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("failed to find content"))
		w.Write([]byte("failed to find posts"))
		return
	}

	e.log.Println("list has been found")
	e.HTML(w, r, ListHTML, map[string]interface{}{
		"List":  l,
		"Aside": a,

M internal/c/search/search.go => internal/c/search/search.go +30 -39
@@ 16,76 16,67 @@ var ItemHTML = tmpl.MustParse("html/item.html")
type SearchEndpoint struct {
	*c.Endpoint
	log                          *log.Logger
	cms                          cmsProvider
	typePage, typePost, typeMeta int
}

type cmsProvider interface {
	Find(ctx context.Context, typeID int, field, query string) (*cms.Content, error)
}

func New(base *c.Endpoint, log *log.Logger, cms cmsProvider, typePage, typePost, typeMeta int) *SearchEndpoint {
func New(base *c.Endpoint, log *log.Logger, typePage, typePost, typeMeta int) *SearchEndpoint {
	return &SearchEndpoint{
		base,
		log,
		cms,
		typePage, typePost, typeMeta,
	}
}

func (e *SearchEndpoint) search(ctx context.Context, slug string) (c *cms.Content, err error) {
	do := func(ctx context.Context, typ int, slug string, ch chan *cms.Content, ech chan error) {
		c, err := e.cms.Find(ctx, typ, "slug", slug)
		if err != nil {
			ech <- err
			return
		}
		ch <- c
	}

	contentCh := make(chan *cms.Content)
	errCh := make(chan error)
func (e *SearchEndpoint) search(ctx context.Context, slug string) (*cms.Content, *cms.Content, error) {
	cch := make(chan *cms.Content, 2)
	ach := make(chan *cms.Content, 1)
	ech := make(chan error, 3)

	go do(ctx, e.typePost, slug, contentCh, errCh)
	go do(ctx, e.typePage, slug, contentCh, errCh)
	go e.Find(ctx, e.typePage, "slug", slug, cch, ech)
	go e.Find(ctx, e.typePost, "slug", slug, cch, ech)
	go e.Find(ctx, e.typeMeta, "slug", "aside", ach, ech)

	errcount := 0
	var (
		content, aside *cms.Content
		errcount       int
	)

	for {
		select {

		case c := <-contentCh:
			return c, nil

		case err := <-errCh:
		case err := <-ech:
			errcount++
			if errcount > 1 {
				return c, err
				return nil, nil, err
			}
			break

		case c := <-cch:
			content = c
			if aside != nil {
				return content, aside, nil
			}
			break

		case a := <-ach:
			aside = a
			if content != nil {
				return content, aside, nil
			}
			break

		}
	}
}

func (e *SearchEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := e.ContextTimeout(r)

	slug := r.URL.Path[1:]
	if slug == "" {
		slug = "home"
	}

	e.log.Println("searching for content with slug of:", slug)

	c, err := e.search(ctx, slug)
	if err != nil {
		e.log.Println(err)
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("failed to find content"))
		return
	}

	a, err := e.cms.Find(ctx, e.typeMeta, "slug", "aside")
	c, a, err := e.search(ctx, slug)
	if err != nil {
		e.log.Println(err)
		w.WriteHeader(http.StatusNotFound)

A internal/s/cms/cms.go => internal/s/cms/cms.go +92 -0
@@ 0,0 1,92 @@
package cms

import (
	"context"
	"errors"
	"fmt"
	"log"
	"net/http"
	"sync"

	"git.sr.ht/~evanj/evanjon.es/pkg/cms"
)

var (
	ErrBadCache = errors.New("value in cache is corrupted")
)

type CachedCMS struct {
	log    *log.Logger
	cms    *cms.CMS
	cache  sync.Map
	secret string
}

func New(l *log.Logger, cms *cms.CMS, secret string) *CachedCMS {
	return &CachedCMS{
		l,
		cms,
		sync.Map{},
		secret,
	}
}

func (cached *CachedCMS) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if cached.secret != r.FormValue("secret") {
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("you cannot do this"))
		cached.log.Println("cache has failed to clear, no valid secret")
		return
	}

	cached.cache.Range(func(key, val interface{}) bool {
		cached.cache.Delete(key)
		return true
	})

	cached.log.Println("cache has been cleared")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte("cache has been cleared"))
}

func (cached *CachedCMS) Find(ctx context.Context, typeID int, field, query string) (*cms.Content, error) {
	key := fmt.Sprintf("find::%d::%s::%s", typeID, field, query)

	item, ok := cached.cache.Load(key)
	if ok {
		content, ok := item.(*cms.Content)
		if ok {
			return content, nil
		}
		return nil, ErrBadCache
	}

	next, err := cached.cms.Find(ctx, typeID, field, query)
	if err != nil {
		return nil, err
	}

	cached.cache.Store(key, next)
	return next, nil
}

func (cached *CachedCMS) List(ctx context.Context, typeID int, order, field string) ([]cms.Content, error) {
	key := fmt.Sprintf("list::%d::%s::%s", typeID, order, field)

	item, ok := cached.cache.Load(key)
	if ok {
		content, ok := item.([]cms.Content)
		if ok {
			return content, nil
		}
		return nil, ErrBadCache
	}

	next, err := cached.cms.List(ctx, typeID, order, field)
	if err != nil {
		return nil, err
	}

	cached.cache.Store(key, next)
	return next, nil
}

A internal/s/cms/cms_test.go => internal/s/cms/cms_test.go +1 -0
@@ 0,0 1,1 @@
package cms_test