~samwhited/blogsync

7246ec4e2b1c330f8fe52012a5b5b3ce600b4fb1 ā€” Sam Whited 1 year, 9 months ago fa8bf55
blogsync: only republish changed posts

Previously any time a post changed while we were previewing the site
every single post would be regenerated. This change allows only the post
that was changed to be removed or re-created as necessary.
2 files changed, 100 insertions(+), 44 deletions(-)

M preview.go
M publish.go
M preview.go => preview.go +60 -24
@@ 19,7 19,9 @@ import (
	"text/template"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/writeas/go-writeas/v2"
	"mellium.im/blogsync/internal/blog"
	"mellium.im/blogsync/internal/browser"
	"mellium.im/cli"
)


@@ 205,7 207,7 @@ https://writefreely.org/
			}
			debug.Printf("logged in as: %+v", authUser)

			_, collections, err := publish(opts, siteConfig, client, logger, debug)
			compiledTmpl, posted, collections, err := publish(opts, siteConfig, client, logger, debug)
			if err != nil {
				return err
			}


@@ 232,20 234,39 @@ https://writefreely.org/
					if !ok {
						return nil
					}
					logger.Printf("event on file watcher: %v", event)
					for _, coll := range collections {
						// TODO: deleting all posts and starting over is easy but slow. Get the
						// previous/new slug (if different) and create/delete pages as necessary just
						// for the file that changed.
						err = deleteAll(coll.Alias, client)
					if ext := filepath.Ext(event.Name); ext != ".md" && ext != ".markdown" {
						debug.Printf("skipping event on non-markdown file %sā€¦", event.Name)
						continue
					}
					debug.Printf("event on file watcher: %v", event)
					switch event.Op {
					case fsnotify.Chmod:
						// Nothing to do here, skip this event.
						continue
					case fsnotify.Remove, fsnotify.Rename:
						posted, err = removePost(event.Name, posted, client)
						if err != nil {
							logger.Printf("error removing post %s: %v", event.Name, err)
						}
						continue
					case fsnotify.Write:
						// Remove and then don't continue, we'll publish it again in just a
						// moment.
						posted, err = removePost(event.Name, posted, client)
						if err != nil {
							logger.Printf("error clearing old posts: %v", err)
							logger.Printf("error removing old post %s before update: %v", event.Name, err)
						}
						// case fsnotify.Create:
						// Nothing to do here, just continue to publishing.
					}

					_, collections, err = publish(opts, siteConfig, client, logger, debug)
					newPost, err := publishPost(event.Name, opts, siteConfig, nil, collections, compiledTmpl, client, logger, debug)
					if err != nil {
						logger.Printf("error republishing posts: %v", err)
						logger.Printf("error publishing new file %s: %v", event.Name, err)
						continue
					}
					if newPost != nil {
						posted = append(posted, *newPost)
					}
				case err, ok := <-watcher.Errors:
					if !ok {


@@ 341,24 362,39 @@ func mkTmp(cfg writeFreelyConfig, debug *log.Logger) (tmpDir string, e error) {
	return tmpDir, nil
}

func deleteAll(collection string, client *writeas.Client) error {
	// The wire.as API documentation doesn't mention any limit or paging, but only
	// 10 posts ever appear to be returned so just iterate until we can't get
	// anymore posts.
	for {
		posts, err := client.GetCollectionPosts(collection)
func decodeMeta(fname string, meta blog.Metadata, debug *log.Logger) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer func() {
		err := f.Close()
		if err != nil {
			return err
			debug.Printf("error closing %s while reading metadata: %v", fname, err)
		}
		p := *posts
		if len(p) == 0 {
			return nil
		}
		for _, post := range p {
			err := client.DeletePost(post.ID, post.Token)
	}()
	header, err := meta.Decode(f)
	if err != nil {
		return err
	}
	if header != blog.HeaderTOML {
		return fmt.Errorf("expected TOML header but found something else, try the convert command")
	}

	return nil
}

func removePost(fname string, posted []minimalPost, client *writeas.Client) ([]minimalPost, error) {
	// Definitely no metadata, don't bother trying to open the file.
	for i, post := range posted {
		if post.filename == fname {
			err := client.DeletePost(post.id, post.token)
			if err != nil {
				return err
				return posted, err
			}
			posted = append(posted[:i], posted[i+1:]...)
			return posted, err
		}
	}
	return posted, nil
}

M publish.go => publish.go +40 -20
@@ 43,6 43,13 @@ type publishOptions struct {
	tmpl              string
}

type minimalPost struct {
	filename string
	id       string
	slug     string
	token    string
}

func newPublishOpts(siteConfig Config) publishOptions {
	return publishOptions{
		collection: siteConfig.Collection,


@@ 69,14 76,15 @@ func publishCmd(siteConfig Config, client *writeas.Client, logger, debug *log.Lo
Expects an API token to be exported as $%s.`, envToken),
		Flags: flags,
		Run: func(cmd *cli.Command, args ...string) error {
			_, _, err := publish(opts, siteConfig, client, logger, debug)
			_, _, _, err := publish(opts, siteConfig, client, logger, debug)
			return err
		},
	}
}

func publish(opts publishOptions, siteConfig Config, client *writeas.Client, logger, debug *log.Logger) (*template.Template, []writeas.Collection, error) {
func publish(opts publishOptions, siteConfig Config, client *writeas.Client, logger, debug *log.Logger) (*template.Template, []minimalPost, []writeas.Collection, error) {
	var collections []writeas.Collection

	if opts.createCollections {
		colls, err := client.GetUserCollections()
		if err != nil {


@@ 101,7 109,7 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
		// should load.
		compiledTmpl, err = compiledTmpl.ParseFiles(tmplFile)
		if err != nil {
			return nil, nil, fmt.Errorf("error compiling template file %s: %v", tmplFile, err)
			return nil, nil, nil, fmt.Errorf("error compiling template file %s: %v", tmplFile, err)
		}
		compiledTmpl = compiledTmpl.Lookup(tmplFile)
	} else {


@@ 109,14 117,14 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
		// Otherwise, it is a raw template and we should compile it.
		compiledTmpl, err = compiledTmpl.Parse(opts.tmpl)
		if err != nil {
			return nil, nil, fmt.Errorf("error compiling template: %v", err)
			return nil, nil, nil, fmt.Errorf("error compiling template: %v", err)
		}
	}

	var posts []writeas.Post
	p, err := client.GetUserPosts()
	if err != nil {
		return nil, nil, fmt.Errorf("error fetching users posts: %v", err)
		return nil, nil, nil, fmt.Errorf("error fetching users posts: %v", err)
	}
	// For now, the writeas SDK returns things with a lot of unnecessary
	// indirection that makes the library hard to use.


@@ 125,11 133,16 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
	// See: https://github.com/writeas/go-writeas/pull/19
	posts = *p

	posted := make([]minimalPost, 0, len(posts))
	err = blog.WalkPages(opts.content, func(pagePath string, info os.FileInfo, err error) error {
		return publishPost(pagePath, opts, siteConfig, posts, collections, compiledTmpl, client, logger, debug)
		newPost, err := publishPost(pagePath, opts, siteConfig, posts, collections, compiledTmpl, client, logger, debug)
		if newPost != nil {
			posted = append(posted, *newPost)
		}
		return err
	})
	if err != nil {
		return nil, nil, err
		return nil, nil, nil, err
	}

	// Delete remaining posts for which we couldn't find a matching file.


@@ 147,15 160,15 @@ func publish(opts publishOptions, siteConfig Config, client *writeas.Client, log
		logger.Printf("no file found matching post %q, re-run with --delete to remove", post.Slug)
	}

	return compiledTmpl, collections, nil
	return compiledTmpl, posted, collections, nil
}

func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts []writeas.Post, collections []writeas.Collection, compiledTmpl *template.Template, client *writeas.Client, logger, debug *log.Logger) error {
func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts []writeas.Post, collections []writeas.Collection, compiledTmpl *template.Template, client *writeas.Client, logger, debug *log.Logger) (post *minimalPost, err error) {
	debug.Printf("opening %s", pagePath)
	fd, err := os.Open(pagePath)
	if err != nil {
		logger.Printf("error opening %s, skipping: %v", pagePath, err)
		return nil
		return nil, nil
	}
	defer func() {
		if err := fd.Close(); err != nil {


@@ 168,26 181,26 @@ func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts 
	header, err := meta.Decode(f)
	if err != nil {
		logger.Printf("error decoding metadata for %s, skipping: %v", pagePath, err)
		return nil
		return nil, nil
	}
	// This may seem unnecessary, but I don't plan on supporting YAML
	// headers forever to keep things simple, so go ahead and forbid
	// publishing with them to encourage people to convert their blogs over.
	if header == blog.HeaderYAML {
		logger.Printf(`file %s has a YAML header, try converting it by running "%s convert", skipping`, pagePath, os.Args[0])
		return nil
		return nil, nil
	}

	draft := meta.GetBool("draft")
	if draft {
		debug.Printf("skipping draft %s", pagePath)
		return nil
		return nil, nil
	}

	title := meta.GetString("title")
	if title == "" {
		logger.Printf("invalid or empty title in %s, skipping", pagePath)
		return nil
		return nil, nil
	}

	// Deliberately shadow collection so that we don't end up mutating the


@@ 200,7 213,7 @@ func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts 
	body, err := ioutil.ReadAll(f)
	if err != nil {
		logger.Printf("error reading body from %s, skipping: %v", pagePath, err)
		return nil
		return nil, nil
	}
	body = bytes.TrimSpace(body)
	body = blackfriday.Run(body,


@@ 223,12 236,12 @@ func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts 
	})
	if err != nil {
		logger.Printf("error executing template for file %s: %v", pagePath, err)
		return nil
		return nil, nil
	}
	if bodyBuf.Len() == 0 {
		// Apparently write.as doesn't like posts that don't have a body.
		logger.Printf("post %s has no body, skipping", pagePath)
		return nil
		return nil, nil
	}

	slug := blog.Slug(pagePath, meta)


@@ 302,9 315,10 @@ func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts 
			post, err := client.CreatePost(params)
			if err != nil {
				logger.Printf("error creating post from %s: %v", pagePath, err)
				return nil
				return nil, nil
			}
			postID = post.ID
			postTok = post.Token
		} else {
			// Write.as returns a generic 500 error if you set Created when
			// updating a post, even if it's unchanged.


@@ 312,9 326,10 @@ func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts 
			post, err := client.UpdatePost(postID, postTok, params)
			if err != nil {
				logger.Printf("error updating post %q from %s: %v", postID, pagePath, err)
				return nil
				return nil, nil
			}
			postID = post.ID
			postTok = post.Token
		}
	}



@@ 347,7 362,12 @@ func publishPost(pagePath string, opts publishOptions, siteConfig Config, posts 
		}
	}

	return nil
	return &minimalPost{
		filename: pagePath,
		slug:     slug,
		id:       postID,
		token:    postTok,
	}, nil
}

func createCollectionIfNotExist(colls []writeas.Collection, client *writeas.Client, debug *log.Logger, coll *writeas.CollectionParams) []writeas.Collection {