~artemis/cap

8848a3358aed317b8e272131c37eebcf77841ae1 — Artémis 1 year, 7 months ago aa97cb8
Removed full image processing and per-image config

Maintaining both side-by-side was annoyingly complex for no reason, and
the per-image TOML configuration file was also a burden to maintain.

I manually compress the images before integrating them to my capsule's
git tree to avoid storing large blobs that would need to be reprocessed
on every new build, and I don't use the per-file configuration system,
only the global thumbnail configuration (which I leave to its defaults,
even).

I wish to remove the immensely big codebundle handling the async part
too, it feels really overkill and not useful at all.
I want to keep the async nature of the media processing, but not in its
current state.
6 files changed, 31 insertions(+), 94 deletions(-)

M .vscode/launch.json
M README.md
M builder.go
M config.go
M handler.go
M types.go
M .vscode/launch.json => .vscode/launch.json +2 -2
@@ 9,7 9,7 @@
			"type": "go",
			"request": "launch",
			"mode": "auto",
			"args": ["-p", "../aphrodite", "build", "-v"],
			"args": ["-p", "../aphrodite.dev", "build", "-v"],
			"program": "${workspaceFolder}"
		},
		{


@@ 17,7 17,7 @@
			"type": "go",
			"request": "launch",
			"mode": "auto",
			"args": ["-p", "../aphrodite", "watch", "-v"],
			"args": ["-p", "../aphrodite.dev", "watch", "-v"],
			"program": "${workspaceFolder}"
		},
		{

M README.md => README.md +1 -1
@@ 43,7 43,7 @@ What are the differences between both?

-   External build scripts
    -   Zola is a one-shot builder not including any "external buildscript" support
    -   Cap has a configuration property (`postbuildcommands`) made to provide an ordered list of commands you want to run after Cap has built the blog
    -   Cap has a configuration property made to provide an ordered list of commands you want to run after Cap has built the blog
    -   The why: I have this need for my blog, because I have a post-build script that modifies every HTML file to set a custom theme value
-   Gemini support
    -   Zola is markdown-based, and doesn't have in its scopes to support Gemini

M builder.go => builder.go +9 -14
@@ 28,9 28,10 @@ func Build(root string, args *Args) (*Config, error) {
	}
	domain.L.Debugf("Config loaded:\n%v", cfg)
	if args.DisableImageOptimization {
		domain.L.Info("Image optimization has been disabled for both images and thumbnails")
		cfg.Images.Disable = true
		domain.L.Info("Thumbnails optimization has been disabled")
		cfg.Thumbnails.Disable = true
	} else if _, err := exec.LookPath("convert"); err != nil {
		domain.L.Warn("`convert` not found in path. Did you install ImageMagick? Disabling thumbnail optimization")
	}

	// 2. Creating file index


@@ 67,27 68,21 @@ func Build(root string, args *Args) (*Config, error) {
		}
	}

	fb.MediaPool = mediapool.InitAndStartAsyncMediaPool(func(id int, wg *sync.WaitGroup, mediaPipeline <-chan *domain.ContentFile, errorPipeline chan<- error) {
	fb.ThumbnailPool = mediapool.InitAndStartAsyncMediaPool(func(id int, wg *sync.WaitGroup, mediaPipeline <-chan *domain.ContentFile, errorPipeline chan<- error) {
		defer wg.Done()

		for media := range mediaPipeline {
			domain.L.Debugf("[Worker #%d] Handling media %s", id, media.RelativePath)
			domain.L.Debugf("[Worker #%d] Generating thumbnail for media %s", id, media.RelativePath)

			globalConfigToUse := map[domain.ContentFileType]OptimizationParameters{
				domain.ContentFileTypeUnused:    fb.Config.Images,
				domain.ContentFileTypeMedia:     fb.Config.Images,
				domain.ContentFileTypeThumbnail: fb.Config.Thumbnails,
			}[media.Kind]

			// Ensuring the folder exists, since we're breaking the object tree system
			// Ensuring the folder exists, since we're breaking the ordered object tree system
			targetDir := filepath.Dir(fb.GetPath(media.RelativeOutputPath, true))
			if err := os.MkdirAll(targetDir, 0755); err != nil {
				errorPipeline <- err
				continue
			}

			if err := fb.HandleMedia(globalConfigToUse, media); err != nil {
				domain.L.Debugf("[Worker #%d] Failed handling media %s with error\n%v", id, media.RelativePath, err)
			if err := fb.HandleMedia(fb.Config.Thumbnails, media); err != nil {
				domain.L.Debugf("[Worker #%d] Failed generating thumbnail for media %s with error\n%v", id, media.RelativePath, err)
				errorPipeline <- err
			}
		}


@@ 120,7 115,7 @@ func Build(root string, args *Args) (*Config, error) {
	}

	// 7a. Check on (and wait for) the media processing pipeline
	err = fb.MediaPool.WaitForFinish()
	err = fb.ThumbnailPool.WaitForFinish()
	if err != nil {
		return cfg, err
	}

M config.go => config.go +0 -12
@@ 30,13 30,6 @@ func LoadConfig(path string) (*Config, error) {
	}

	outputCfg := Config{
		Images: OptimizationParameters{
			KeepMetadata:       false,
			ResizeRatio:        "x720",
			DisableProgressive: false,
			Quality:            85,
			SamplingFactor:     "4:2:0",
		},
		Thumbnails: OptimizationParameters{
			KeepMetadata:       false,
			ResizeRatio:        "x240",


@@ 58,11 51,6 @@ func LoadConfig(path string) (*Config, error) {
			outputCfg.PostBuild.Commands = append(outputCfg.PostBuild.Commands, outputCfg.PostBuildCommands...)
		}
	}
	if outputCfg.DisableImageOptimization {
		domain.L.Warn("The disable_image_optimization property is deprecated in favor of the images.disable property.\nThe deprecated one will also disable optimization for thumbnails.")
		outputCfg.Images.Disable = outputCfg.DisableImageOptimization
		outputCfg.Thumbnails.Disable = outputCfg.DisableImageOptimization
	}

	if err = validator.New().Struct(outputCfg); err != nil {
		return nil, fmt.Errorf("in the capsule's config file,\n%s", err.Error())

M handler.go => handler.go +7 -51
@@ 17,8 17,6 @@ import (
	"git.sr.ht/~artemis/cap/domain"
	"github.com/eduncan911/podcast"
	"github.com/google/shlex"
	"github.com/imdario/mergo"
	"github.com/pelletier/go-toml/v2"
)

// HandleAssets builds the output folder for raw assets, and then copies them all over.


@@ 77,16 75,10 @@ func (f *FsBuilder) HandleSection(section *domain.Section) (int, error) {
		}
	}

	if f.Config.Images.Disable {
		// If disabled, handle the medias as files
		for _, media := range section.Media {
			if err := f.HandleFile(media); err != nil {
				return 0, err
			}
	for _, media := range section.Media {
		if err := f.HandleFile(media); err != nil {
			return 0, err
		}
	} else {
		// If enabled, handle the medias as images, through multiple workers
		f.MediaPool.AddMedias(section.Media)
	}

	if section.Page != nil {


@@ 96,9 88,9 @@ func (f *FsBuilder) HandleSection(section *domain.Section) (int, error) {
		}

		if section.Page.ThumbnailFile != nil {
			domain.L.Debugf("Found thumbnail for page %s, handling as media", section.Page.Permalink)
			domain.L.Debugf("Found thumbnail for page %s, marking for compression", section.Page.Permalink)

			f.MediaPool.AddMedias([]*domain.ContentFile{section.Page.ThumbnailFile})
			f.ThumbnailPool.AddMedias([]*domain.ContentFile{section.Page.ThumbnailFile})
		}
	}
	for _, page := range section.Pages {


@@ 110,7 102,7 @@ func (f *FsBuilder) HandleSection(section *domain.Section) (int, error) {
		if page.ThumbnailFile != nil {
			domain.L.Debugf("Found thumbnail for page %s, handling as media", page.Permalink)

			f.MediaPool.AddMedias([]*domain.ContentFile{page.ThumbnailFile})
			f.ThumbnailPool.AddMedias([]*domain.ContentFile{page.ThumbnailFile})
		}
	}



@@ 253,8 245,7 @@ func (f *FsBuilder) HandleFeed(feed *domain.Feed) error {

// HandleMedia takes a media file, and tried to render it using imagemagick's convert command.
func (f *FsBuilder) HandleMedia(optimizationParameters OptimizationParameters, media *domain.ContentFile) error {
	shouldBeIgnored := optimizationParameters.Disable ||
		strings.HasPrefix(filepath.Base(media.Path), "raw.")
	shouldBeIgnored := optimizationParameters.Disable

	if shouldBeIgnored {
		return f.HandleFile(media)


@@ 269,21 260,6 @@ func (f *FsBuilder) HandleMedia(optimizationParameters OptimizationParameters, m

	domain.L.Debug("`convert` found, running it against the file")

	configPath := media.Path + ".toml"
	if _, err := os.Stat(configPath); err == nil {
		loadedParams, err := LoadOptimizationParametersFromFile(configPath)
		if err != nil {
			return fmt.Errorf(
				"tried to load optimization parameters from the configuration file %s, for the image at %s, but failed\n%s",
				media.RelativePath+".toml",
				media.RelativePath,
				err.Error(),
			)
		}

		mergo.Merge(&optimizationParameters, *loadedParams, mergo.WithOverride)
	}

	if err := f.HandleMediaCompression(media, optimizationParameters); err != nil {
		domain.L.Warnf("Compression failed for image %s, copying as-is instead", media.RelativePath)
		if err == context.DeadlineExceeded {


@@ 298,26 274,6 @@ func (f *FsBuilder) HandleMedia(optimizationParameters OptimizationParameters, m
	return nil
}

// LoadOptimizationParametersFromFile loads parameters from an image-specific configuration file.
// TODO: add default passing handling instead of singleton
func LoadOptimizationParametersFromFile(path string) (*OptimizationParameters, error) {
	cfg := &OptimizationParameters{}

	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer fd.Close()

	data, _ := io.ReadAll(fd)
	err = toml.Unmarshal(data, cfg)
	if err != nil {
		return nil, err
	}

	return cfg, nil
}

// HandleMediaCompression takes the media, its configuration, and lets imagemagick's convert do its magic with image resizing and compression.
func (f *FsBuilder) HandleMediaCompression(media *domain.ContentFile, optimizationParameters OptimizationParameters) error {
	from := f.GetPath(media.RelativePath, false)

M types.go => types.go +12 -14
@@ 37,12 37,12 @@ type FileListOpts struct {

// FsBuilder contains the state of the capsule builder objects.
type FsBuilder struct {
	InputPath  string
	OutputPath string
	Config     *Config
	Templates  *template.Template
	Index      *domain.Index
	MediaPool  *mediapool.AsyncMediaPool
	InputPath     string
	OutputPath    string
	Config        *Config
	Templates     *template.Template
	Index         *domain.Index
	ThumbnailPool *mediapool.AsyncMediaPool
}

// OptimizationParameters defines all optimization parameters the user can configure for tweaking and fine-tuning the media compression.


@@ 78,14 78,12 @@ type PostBuildConfig struct {

// Config stores the capsule's configuration, including metadata, media optimization info, post-build commands, etc.
type Config struct {
	Capsule                  CapsuleConfig
	Images                   OptimizationParameters
	Thumbnails               OptimizationParameters
	IncludeSource            string          `toml:"include_source" validate:"omitempty,oneof=none gemini all"`
	DisableImageOptimization bool            `toml:"disable_image_optimization"`
	PostBuildCommands        []string        `toml:"post_build_commands"`
	PostBuild                PostBuildConfig `toml:"post_build"`
	FoldersToIgnore          []string        `toml:"ignore"`
	Capsule           CapsuleConfig
	Thumbnails        OptimizationParameters
	IncludeSource     string          `toml:"include_source" validate:"omitempty,oneof=none gemini all"`
	PostBuildCommands []string        `toml:"post_build_commands"`
	PostBuild         PostBuildConfig `toml:"post_build"`
	FoldersToIgnore   []string        `toml:"ignore"`
}

// WatchCMD contains the subcommand's flags and arguments