~whereswaldon/trideo

fba55fb71be97bfbad08c0e5b45cd6003e2774ec — Chris Waldon 2 years ago
feat: add working initial implementation

Signed-off-by: Chris Waldon <christopher.waldon.dev@gmail.com>
3 files changed, 419 insertions(+), 0 deletions(-)

A go.mod
A go.sum
A main.go
A  => go.mod +15 -0
@@ 1,15 @@
module git.sr.ht/~whereswaldon/trideo

go 1.16

require (
	gioui.org v0.0.0-20210410094005-495c69018772
	git.sr.ht/~whereswaldon/latest v0.0.0-20210308171958-a201efcdbede
	github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539
	github.com/esimov/triangle v1.1.1
	github.com/pkg/profile v1.5.0
	golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb // indirect
	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
)

replace github.com/esimov/triangle => github.com/whereswaldon/triangle v1.1.2-0.20210412012813-5280c4b6cb2c

A  => go.sum +43 -0
@@ 1,43 @@
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
gioui.org v0.0.0-20210410094005-495c69018772 h1:QpGMsubuP4JVlZj5VvvFfRikbLDN0QJj71oX9ABeaqM=
gioui.org v0.0.0-20210410094005-495c69018772/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
git.sr.ht/~whereswaldon/latest v0.0.0-20210308171958-a201efcdbede h1:Rw5+/HuphQbIPjlwe7OknvfFSnRp4wBB11PcZ6AeRsg=
git.sr.ht/~whereswaldon/latest v0.0.0-20210308171958-a201efcdbede/go.mod h1:jUOuersHq1652DgWF5rTCCH/Xoepj8z+NpU09IAeqCw=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539 h1:1aIqYfg9s9RETAJHGfVKZW4ok0b22p4QTwk8MsdRtPs=
github.com/blackjack/webcam v0.0.0-20200313125108-10ed912a8539/go.mod h1:G0X+rEqYPWSq0dG8OMf8M446MtKytzpPjgS3HbdOJZ4=
github.com/fogleman/gg v1.0.0 h1:O2ToZn8ijCP2gXhVY701P1b1jrxKoVPh6CkaX2/PACE=
github.com/fogleman/gg v1.0.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug=
github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
github.com/whereswaldon/triangle v1.1.2-0.20210412012813-5280c4b6cb2c h1:/p6V2KLUebKMqRUngxSrKlw96jgOb9BAyhh8Grttt1s=
github.com/whereswaldon/triangle v1.1.2-0.20210412012813-5280c4b6cb2c/go.mod h1:9ERDTfRx/FXTfQbWVXwznmCF+6BhvfbaWVBv/NvRrrA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs=
golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE=
golang.org/x/image v0.0.0-20171214225156-12117c17ca67/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk=
golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

A  => main.go +361 -0
@@ 1,361 @@
package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"image/color"
	_ "image/jpeg"
	"log"
	"os"
	"time"

	"git.sr.ht/~whereswaldon/latest"
	//	"golang.org/x/image/draw"

	"gioui.org/app"
	"gioui.org/f32"
	"gioui.org/io/key"
	"gioui.org/io/system"
	"gioui.org/layout"
	"gioui.org/op"
	"gioui.org/op/clip"
	"gioui.org/op/paint"
	"gioui.org/unit"
	"gioui.org/widget"
	"github.com/blackjack/webcam"
	"github.com/esimov/triangle"
	"github.com/pkg/profile"
)

func main() {
	var useProfiling bool
	flag.BoolVar(&useProfiling, "profile", false, "enable pprof profiling")
	go func() {
		w := app.NewWindow()
		if err := loop(w, useProfiling); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}()
	app.Main()
}

// dump prints a JSONified struct to stdout.
func dump(in interface{}) {
	b, _ := json.MarshalIndent(in, "", "  ")
	fmt.Println(string(b))
}

// Timings holds performance timing information from throughout the render
// process.
type Timings struct {
	Start, Capture, Decode, Convert, Blur, Gray, Sobel, Point, Delaunay time.Time
}

func (t Timings) String() string {
	return fmt.Sprintln("tot:", t.Delaunay.Sub(t.Start))
}

// Result contains the result of rendering a single frame of video
type Result struct {
	// Triangles holds the list of Delaunay triangles representing the frame.
	Triangles []triangle.Triangle
	// Frame is the source image data.
	Frame *image.NRGBA
	// Timings is the performance information from generating this frame.
	Timings
	// Valid indicates whether or not this Result's data can be used. This will
	// be false if something interrupted the render process.
	Valid bool
}

// DebugResult holds all of the intermediate render data.
type DebugResult struct {
	Raw       *image.NRGBA
	Blur      *image.NRGBA
	Sobel     *image.NRGBA
	Points    []triangle.Point
	Triangles []triangle.Triangle
}

// FrameConfig holds the parameters provided to the renderer.
type FrameConfig struct {
	Debug bool
}

// MJPEGFourCC is the 4-byte code corresponding to the MJPEG codec in
// Video4Linux.
const MJPEGFourCC = 1196444237

// loops runs the webcam output UI.
func loop(w *app.Window, useProfiling bool) error {
	logOut := bufio.NewWriter(log.Writer())
	log.SetOutput(logOut)
	defer logOut.Flush()
	if useProfiling {
		defer profile.Start().Stop()
	}
	cam, err := webcam.Open("/dev/video0")
	if err != nil {
		panic(err)
	}
	defer cam.Close()
	if _, supported := cam.GetSupportedFormats()[MJPEGFourCC]; !supported {
		panic(fmt.Errorf("webcam doesn't support MJPEG"))
	}
	sizes := cam.GetSupportedFrameSizes(MJPEGFourCC)
	target := 0
	for index, config := range sizes {
		if config.MinWidth < sizes[target].MinWidth {
			target = index
		}
	}
	dump(sizes[target])
	format, width, height, err := cam.SetImageFormat(MJPEGFourCC, sizes[target].MaxWidth, sizes[target].MinHeight)
	if err != nil {
		panic(err)
	}
	log.Println("webcam confirms", format, width, height)
	if err := cam.SetBufferCount(1); err != nil {
		panic(err)
	}

	if err := cam.StartStreaming(); err != nil {
		panic(err)
	}

	p := triangle.Processor{
		BlurRadius:      4,
		SobelThreshold:  10,
		PointsThreshold: 20,
		MaxPoints:       500,
		StrokeWidth:     0,
		Wireframe:       0,
	}

	var ops op.Ops
	tickerDuration := time.Second / 15
	ticker := time.NewTicker(tickerDuration)
	var init bool
	var d triangle.Delaunay
	var tris Result
	worker := latest.NewWorker(func(in interface{}) interface{} {
		config := in.(FrameConfig)
		var timings Timings
		timings.Start = time.Now()
		if err := cam.WaitForFrame(1); err != nil {
			return tris
		}
		buf, err := cam.ReadFrame()
		if err != nil {
			return tris
		}
		timings.Capture = time.Now()
		frame, _, err := image.Decode(bytes.NewBuffer(buf))
		if err != nil {
			return tris
		}
		timings.Decode = time.Now()
		width, height := frame.Bounds().Dx(), frame.Bounds().Dy()
		asRGBA := triangle.ToNRGBA(frame)
		timings.Convert = time.Now()
		blur := asRGBA
		if p.BlurRadius > 0 {
			blur = triangle.StackBlur(asRGBA, uint32(p.BlurRadius))
		}
		timings.Blur = time.Now()
		sobel := triangle.SobelFilter(blur, float64(p.SobelThreshold))
		timings.Sobel = time.Now()
		points := triangle.GetEdgePoints(sobel, p.PointsThreshold, p.MaxPoints)
		timings.Point = time.Now()

		triangles := d.Init(width, height).Insert(points).GetTriangles()
		timings.Delaunay = time.Now()
		if config.Debug {
			go DebugWindow(DebugResult{
				Raw:       asRGBA,
				Blur:      blur,
				Sobel:     sobel,
				Points:    points,
				Triangles: triangles,
			})
		}
		return Result{Triangles: triangles, Frame: asRGBA, Timings: timings, Valid: true}

	})
	var aspect float32
	var debug bool
	for {
		select {
		case img := <-worker.Raw():
			tris = img.(Result)
			if tris.Valid {
				if !init {
					aspect = float32(tris.Frame.Bounds().Dx()) / float32(tris.Frame.Bounds().Dy())
				}
				init = true
				w.Invalidate()
			}
		case <-ticker.C:
			worker.Push(FrameConfig{Debug: debug})
			debug = false
		case e := <-w.Events():
			switch e := e.(type) {
			case system.DestroyEvent:
				return e.Err
			case system.FrameEvent:
				start := time.Now()
				sz := layout.FPt(e.Size)
				szAspect := sz.X / sz.Y
				if init && (szAspect-aspect > 0.01 || szAspect-aspect < -0.01) {
					width, height := unit.Dp(sz.Y*aspect), unit.Dp(sz.Y)
					log.Println(aspect, szAspect, width, height)
					w.Option(app.Size(width, height))
					w.Invalidate()
				}
				gtx := layout.NewContext(&ops, e)
				if init {
					paint.Fill(gtx.Ops, color.NRGBA{A: 255})
					op.Affine(f32.Affine2D{}.Scale(f32.Point{}, f32.Point{
						X: float32(gtx.Constraints.Max.X) / float32(tris.Frame.Rect.Dx()),
						Y: float32(gtx.Constraints.Max.Y) / float32(tris.Frame.Rect.Dy()),
					})).Add(gtx.Ops)
					for _, t := range tris.Triangles {
						LayoutDelaunyTriangle(gtx, t, tris.Frame)
					}
				}
				e.Frame(gtx.Ops)
				now := time.Now()
				log.Println("frame", now.Sub(start), "latency", now.Sub(tris.Start))
			case key.Event:
				if e.Name == key.NameSpace {
					debug = true
				}
			}
		}
	}
}

// Dp converts an integer quantity into unit.Dp.
func Dp(in int) unit.Value {
	return unit.Dp(float32(in))
}

// DebugWindow launches a new Gio window to present the DebugResult for
// visual inspection.
func DebugWindow(result DebugResult) {
	width := result.Raw.Rect.Dx() * 4
	height := result.Raw.Rect.Dy() * 4
	w := app.NewWindow(app.Title("Debug"), app.Size(Dp(width), Dp(height*5)))
	var ops op.Ops
	var list layout.List
	list.Axis = layout.Vertical
	for {
		select {
		case e := <-w.Events():
			switch e := e.(type) {
			case system.DestroyEvent:
				return
			case system.FrameEvent:
				gtx := layout.NewContext(&ops, e)
				list.Layout(gtx, 5, func(gtx layout.Context, index int) layout.Dimensions {
					switch index {
					case 0:
						return LayoutImage(gtx, result.Raw)
					case 1:
						return LayoutImage(gtx, result.Blur)
					case 2:
						return LayoutImage(gtx, result.Sobel)
					case 3:
						return LayoutPointsOnImage(gtx, result.Sobel, result.Points)
					case 4:
						return LayoutTrianglesOnImage(gtx, result.Sobel, result.Triangles)
					default:
						return layout.Dimensions{}
					}
				})
				e.Frame(gtx.Ops)
			}

		}
	}
}

// LayoutPointsOnImage lays out an image with some points superimposed on
// top of it.
func LayoutPointsOnImage(gtx layout.Context, img image.Image, points []triangle.Point) layout.Dimensions {
	dims := LayoutImage(gtx, img)
	scale := float32(dims.Size.X) / float32(img.Bounds().Dx())
	defer op.Save(gtx.Ops).Load()
	op.Affine(f32.Affine2D{}.Scale(f32.Point{}, f32.Point{X: scale, Y: scale})).Add(gtx.Ops)
	for _, p := range points {
		pt := f32.Point{X: float32(p.X), Y: float32(p.Y)}
		DrawPoint(gtx, pt)
	}
	return dims
}

// DrawPoint renders a small green point at the provided location.
func DrawPoint(gtx layout.Context, p f32.Point) {
	paint.FillShape(gtx.Ops, color.NRGBA{G: 255, A: 255},
		clip.Circle{
			Center: p,
			Radius: 0.3,
		}.Op(gtx.Ops))
}

// LayoutTrianglesOnImage renders a image with triangles and their center
// points indicated on top of it.
func LayoutTrianglesOnImage(gtx layout.Context, img image.Image, tris []triangle.Triangle) layout.Dimensions {
	dims := LayoutImage(gtx, img)
	scale := float32(dims.Size.X) / float32(img.Bounds().Dx())
	defer op.Save(gtx.Ops).Load()
	op.Affine(f32.Affine2D{}.Scale(f32.Point{}, f32.Point{X: scale, Y: scale})).Add(gtx.Ops)
	for _, t := range tris {
		path, center := TriPath(gtx, t)
		outline := clip.Stroke{
			Path: path,
			Style: clip.StrokeStyle{
				Width: 0.1,
			},
		}.Op()
		paint.FillShape(gtx.Ops, color.NRGBA{G: 255, A: 255}, outline)
		DrawPoint(gtx, layout.FPt(center))
	}
	return dims
}

// LayoutImage displays an image contained by the gtx constraints.
func LayoutImage(gtx layout.Context, img image.Image) layout.Dimensions {
	imgOp := paint.NewImageOp(img)
	return widget.Image{
		Src: imgOp,
		Fit: widget.Contain,
	}.Layout(gtx)
}

// TriPath returns the path representing the given triangle as well as the
// triangle's center point.
func TriPath(gtx layout.Context, t triangle.Triangle) (clip.PathSpec, image.Point) {
	p := clip.Path{}
	p.Begin(gtx.Ops)
	a, b, c := t.Nodes[0], t.Nodes[1], t.Nodes[2]
	cx := (a.X + b.X + c.X) / 3
	cy := (a.Y + b.Y + c.Y) / 3
	p.MoveTo(layout.FPt(image.Pt(a.X, a.Y)))
	p.LineTo(layout.FPt(image.Pt(b.X, b.Y)))
	p.LineTo(layout.FPt(image.Pt(c.X, c.Y)))
	p.Close()
	return p.End(), image.Pt(cx, cy)
}

// LayoutDelaunyTriangle renders a single colored triangle.
func LayoutDelaunyTriangle(gtx layout.Context, t triangle.Triangle, frame *image.NRGBA) {
	path, center := TriPath(gtx, t)
	outline := clip.Outline{Path: path}.Op()
	paint.FillShape(gtx.Ops, frame.At(center.X, center.Y).(color.NRGBA), outline)
}