~samwhited/blogsync

e1b81f3a2ecc9b4331bcc1c62ff7a3723975f9cd — Sam Whited 1 year, 9 months ago 79ae993
blogsync: add naive filesystem watching

This implementation currently deletes all posts and re-uploads them any
time a source file changes. This is naive, slow, and requires a *lot* of
API requests, but for now it works to show the concept.
The next step will be to figure out the event type and the old and new
slugs and only change the files and posts that need to be tweaked.

Fixes #4
5 files changed, 128 insertions(+), 20 deletions(-)

M go.mod
M go.sum
M preview.go
M publish.go
A watch.go
M go.mod => go.mod +1 -0
@@ 4,6 4,7 @@ go 1.13

require (
	github.com/BurntSushi/toml v0.3.1
	github.com/fsnotify/fsnotify v1.4.7
	github.com/pmezard/go-difflib v1.0.0 // indirect
	github.com/russross/blackfriday/v2 v2.0.1
	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect

M go.sum => go.sum +2 -0
@@ 2,6 2,8 @@ code.as/core/socks v1.0.0 h1:SPQXNp4SbEwjOAP9VzUahLHak8SDqy5n+9cm9tpjZOs=
code.as/core/socks v1.0.0/go.mod h1:BAXBy5O9s2gmw6UxLqNJcVbWY7C/UPs+801CcSsfWOY=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=

M preview.go => preview.go +71 -13
@@ 92,16 92,18 @@ default_visibility = public
`

func previewCmd(siteConfig Config, logger, debug *log.Logger) *cli.Command {
	opts := newPublishOpts(siteConfig)
	opts.createCollections = true

	var (
		port    = 8080
		bind    = "127.0.0.1"
		content = "content/"
		res     = "/usr/share/writefreely/"
		port = 8080
		bind = "127.0.0.1"
		res  = "/usr/share/writefreely/"
	)
	flags := flag.NewFlagSet("preview", flag.ContinueOnError)
	flags.IntVar(&port, "port", port, "The port for writefreely to bind to")
	flags.StringVar(&bind, "addr", bind, "The address the server should bind to")
	flags.StringVar(&content, "content", content, "A directory containing pages and posts")
	flags.StringVar(&opts.content, "content", opts.content, "A directory containing pages and posts")
	flags.StringVar(&res, "resources", res, "A directory containing writefreelys templates and static assets")

	return &cli.Command{


@@ 203,21 205,55 @@ https://writefreely.org/
			}
			debug.Printf("logged in as: %+v", authUser)

			opts := newPublishOpts(siteConfig)
			opts.createCollections = true

			err = publish(opts, siteConfig, client, logger, debug)
			collections, err := publish(opts, siteConfig, client, logger, debug)
			if err != nil {
				return err
			}

			browser.Open(baseAddr)

			select {
			case <-sigs:
			case <-ctx.Done():
			watcher, err := newWatcher(opts.content, debug)
			if err != nil {
				return fmt.Errorf("error watching %s for changes: %w", opts.content, err)
			}
			defer func() {
				err := watcher.Close()
				if err != nil {
					debug.Printf("error closing %s watcher: %v", opts.content, err)
				}
			}()
			for {
				select {
				case <-sigs:
					return nil
				case <-ctx.Done():
					return nil
				case event, ok := <-watcher.Events:
					if !ok {
						return nil
					}
					logger.Printf("event on file watcher: %v", event)
					for _, coll := range collections {
						// TODO: deleting all posts and starting over is easy but slow. Get the
						// previous/new slug (if different) and create/delete pages as necessary just
						// for the file that changed.
						err = deleteAll(coll.Alias, client)
						if err != nil {
							logger.Printf("error clearing old posts: %v", err)
						}
					}

					collections, err = publish(opts, siteConfig, client, logger, debug)
					if err != nil {
						logger.Printf("error republishing posts: %v", err)
					}
				case err, ok := <-watcher.Errors:
					if !ok {
						return nil
					}
					logger.Printf("error on watcher: %v", err)
				}
			}
			return nil
		},
	}
}


@@ 304,3 340,25 @@ func mkTmp(cfg writeFreelyConfig, debug *log.Logger) (tmpDir string, e error) {

	return tmpDir, nil
}

func deleteAll(collection string, client *writeas.Client) error {
	// The wire.as API documentation doesn't mention any limit or paging, but only
	// 10 posts ever appear to be returned so just iterate until we can't get
	// anymore posts.
	for {
		posts, err := client.GetCollectionPosts(collection)
		if err != nil {
			return err
		}
		p := *posts
		if len(p) == 0 {
			return nil
		}
		for _, post := range p {
			err := client.DeletePost(post.ID, post.Token)
			if err != nil {
				return err
			}
		}
	}
}

M publish.go => publish.go +8 -7
@@ 69,12 69,13 @@ func publishCmd(siteConfig Config, client *writeas.Client, logger, debug *log.Lo
Expects an API token to be exported as $%s.`, envToken),
		Flags: flags,
		Run: func(cmd *cli.Command, args ...string) error {
			return publish(opts, siteConfig, client, logger, debug)
			_, err := publish(opts, siteConfig, client, logger, debug)
			return err
		},
	}
}

func publish(opts publishOptions, siteConfig Config, client *writeas.Client, logger, debug *log.Logger) error {
func publish(opts publishOptions, siteConfig Config, client *writeas.Client, logger, debug *log.Logger) ([]writeas.Collection, error) {
	var collections []writeas.Collection
	if opts.createCollections {
		colls, err := client.GetUserCollections()


@@ 100,21 101,21 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
		// should load.
		compiledTmpl, err = compiledTmpl.ParseFiles(tmplFile)
		if err != nil {
			return fmt.Errorf("error compiling template file %s: %v", tmplFile, err)
			return nil, fmt.Errorf("error compiling template file %s: %v", tmplFile, err)
		}
	} else {
		tmplFile = defTmplName
		// Otherwise, it is a raw template and we should compile it.
		compiledTmpl, err = compiledTmpl.Parse(opts.tmpl)
		if err != nil {
			return fmt.Errorf("error compiling template: %v", err)
			return nil, fmt.Errorf("error compiling template: %v", err)
		}
	}

	var posts []writeas.Post
	p, err := client.GetUserPosts()
	if err != nil {
		return fmt.Errorf("error fetching users posts: %v", err)
		return nil, fmt.Errorf("error fetching users posts: %v", err)
	}
	// For now, the writeas SDK returns things with a lot of unnecessary
	// indirection that makes the library hard to use.


@@ 323,7 324,7 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
		return nil
	})
	if err != nil {
		return err
		return nil, err
	}

	// Delete remaining posts for which we couldn't find a matching file.


@@ 341,7 342,7 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
		logger.Printf("no file found matching post %q, re-run with --delete to remove", post.Slug)
	}

	return nil
	return collections, nil
}

func createCollectionIfNotExist(colls []writeas.Collection, client *writeas.Client, debug *log.Logger, coll *writeas.CollectionParams) []writeas.Collection {

A watch.go => watch.go +46 -0
@@ 0,0 1,46 @@
// Copyright 2019 The Blog Sync Contributors.
// Use of this source code is governed by the BSD 2-clause
// license that can be found in the LICENSE file.

package main

import (
	"log"
	"os"
	"path/filepath"

	"github.com/fsnotify/fsnotify"
)

func newWatcher(content string, debug *log.Logger) (watcher *fsnotify.Watcher, err error) {
	watcher, err = fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	// Handle errors by cleaning up so that the caller can follow Go idioms of
	// checking errors before handling the value (eg. in case an error happens
	// while adding files to the already existing watcher).
	defer func() {
		if err != nil {
			if err := watcher.Close(); err != nil {
				debug.Printf("error closing unused %s watcher: %v", content, err)
			}
		}
	}()

	err = filepath.Walk(content, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			debug.Printf("error watching file %s, changes will not trigger a rebuilt: %v", path, err)
			return nil
		}

		if !info.IsDir() {
			// Watch entire directory trees for changes, not individual files.
			return nil
		}

		return watcher.Add(path)
	})

	return watcher, err
}