~artemis/cap

52d89615c4556b8d682a5e9ac89ff61ba5d3d8fc — Artemis 4 months ago d6ed0e7 feat/the-flattening 
wip: implemented most of loadPage, except parse and thumbnail
lots of progress!
patch browse .tar.gz 
5 files changed, 282 insertions(+), 184 deletions(-)

M domain/types.go
M loader.go
A render/render.go
A render/renderers.go
A transform/load_from_disk/load_page.go

M domain/types.go => domain/types.go +31 -0
@@ 2,7 2,9 @@ package domain

import (
	"net/url"
	"time"

	"git.sr.ht/~artemis/cap/render"
	"git.sr.ht/~artemis/cap/util/permalink"
	"github.com/google/uuid"
)


@@ 15,6 17,7 @@ const (
	LeafTypeFeed
	LeafTypeFile
	LeafTypePage
	LeafTypeMedia
)

// Leaf is the common interface for every leaves found during build


@@ 66,3 69,31 @@ type File struct {
func (f *File) Type() LeafType {
	return LeafTypeFile
}

type Page struct {
	Identity

	Draft    bool
	Template string
	Date     *time.Time

	CWs     []string
	Title   string
	Summary string
	TOC     []render.Heading
	Content string
}

func (f *Page) Type() LeafType {
	return LeafTypePage
}

type Media struct {
	Identity

	SourcePath string
}

func (f *Media) Type() LeafType {
	return LeafTypeMedia
}


M loader.go => loader.go +103 -184
@@ 2,9 2,14 @@ package main

import (
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"git.sr.ht/~artemis/cap/domain"
	"git.sr.ht/~artemis/cap/util"
	"github.com/adrg/frontmatter"
)

// Parsers contains a map of parsers for every supported page content format.


@@ 28,23 33,8 @@ func ListFiles(root string, idx *domain.Index) ([]domain.Leaf, error) {

	// Walking through the project's root
	err := filepath.Walk(root, func(fsPath string, info fs.FileInfo, err error) error {
		if err != nil || info.IsDir() {
			return err
		}
		relativeToRoot, _ := filepath.Rel(root, fsPath)

		// Ignoring folders
		if info.IsDir() {
			return nil
		}

		// fileFormat := check_format.IdentifyFile(fsPath)
		fsFile := domain.File{
			Identity:   domain.NewIdentityFromPath(relativeToRoot, nil),
			SourcePath: fsPath,
		}
		res = append(res, &fsFile)
		return nil
		// this is moved now.

		// 	ext := filepath.Ext(fsPath)
		// 	// If the file is a page (gemini, markdown)


@@ 117,16 107,6 @@ func ListFiles(root string, idx *domain.Index) ([]domain.Leaf, error) {
		// 		return nil
		// 	}

		// 	// If the file is a known media file
		// 	if util.Includes(ext, ".jpg", ".jpeg", ".png", ".webp") {
		// 		currentSection.Media = append(currentSection.Media, &domain.ContentFile{
		// 			Path:         fsPath,
		// 			RelativePath: relativeToRoot,
		// 			Kind:         domain.ContentFileTypeMedia,
		// 		})

		// 		idx.Files[fsPath] = domain.ObjectTypeMedia

		// 		return nil
		// 	}
	})


@@ 136,167 116,106 @@ func ListFiles(root string, idx *domain.Index) ([]domain.Leaf, error) {

// LoadPage parses, and renders, a single source page from their source format into HTML.
func LoadPage(idx *domain.Index, s *domain.Leaf, path string) (*domain.Leaf, error) {
	// // 1. Base loading
	// base := filepath.Base(path)

	// file, err := os.Open(path)
	// if err != nil {
	// 	return nil, err
	// }
	// defer file.Close()

	// var matter domain.PageFrontmatter

	// raw_content, err := frontmatter.Parse(file, &matter)
	// if err != nil {
	// 	return nil, err
	// }

	// // 1b. Thumbnail handling
	// var thumbnail *domain.ContentFile = nil
	// if matter.Thumbnail != nil {
	// 	if ok, relPath, fullPath := util.CheckRelativeUrl(*matter.Thumbnail); ok {
	// 		if util.Includes(filepath.Ext(relPath), domain.ImageMediaExtensions...) {
	// 			thumbnailExt := filepath.Ext(*matter.Thumbnail)
	// 			thumbnailPath := strings.TrimSuffix(*matter.Thumbnail, thumbnailExt) + ".thumb" + thumbnailExt

	// 			thumbnail = &domain.ContentFile{
	// 				Path:               filepath.Join(s.Path, fullPath),
	// 				RelativePath:       filepath.Join(s.RelativePath, fullPath),
	// 				RelativeOutputPath: filepath.Join(s.RelativePath, thumbnailPath),
	// 				Kind:               domain.ContentFileTypeThumbnail,
	// 			}
	// 		} else {
	// 			domain.L.Warnf(
	// 				"In the page at %s,\nthe thumbnail is set to the value `%s`, but the path appears to point to a non-media file...\nAre you sure you are pointing to the right file?\nIgnoring value...",
	// 				path,
	// 				*matter.Thumbnail,
	// 			)
	// 		}
	// 	} else {
	// 		domain.L.Warnf(
	// 			"In the page at %s,\nthe thumbnail is set to the value `%s`, but no file could be found at this path.\nIs it the right path?\nIgnoring value...",
	// 			path,
	// 			*matter.Thumbnail,
	// 		)
	// 	}
	// }

	// content := string(raw_content)

	// // 2. Required metadata building
	// format := Formats[filepath.Ext(base)]
	// sort.Strings(matter.CWs)

	// // 3. Page building
	// page := &domain.Page{
	// 	Section:       s,
	// 	Path:          path,
	// 	Content:       content,
	// 	Draft:         matter.Draft,
	// 	Format:        format,
	// 	Template:      matter.Template,
	// 	Date:          matter.Date,
	// 	Matter:        matter,
	// 	CWs:           matter.CWs,
	// 	ThumbnailFile: thumbnail,
	// }

	// // 4. Rendering
	// rendered, err := Parsers[page.Format].Parse(idx, page)
	// if err != nil {
	// 	return page, err
	// }
	// page.RenderedContent = rendered.Content
	// page.Summary = rendered.Summary
	// page.TOC = rendered.TOC
	// page.LinksCount = rendered.Counts.Links
	// page.MediasCount = rendered.Counts.Medias

	// // 5. Derived metadata application
	// baseWithoutExt := strings.TrimSuffix(base, filepath.Ext(base))
	// if len(rendered.Title) != 0 {
	// 	page.Title = rendered.Title
	// } else if page.Matter.Title != "" {
	// 	page.Title = page.Matter.Title
	// } else {
	// 	page.Title = baseWithoutExt
	// }
	// page.Slug = baseWithoutExt
	// if page.Slug == "index" {
	// 	page.Permalink = page.Section.Permalink
	// } else {
	// 	page.Permalink = filepath.Join(page.Section.Permalink, page.Slug+".html")
	// }
	// if page.ThumbnailFile != nil {
	// 	page.Thumbnail = "/" + page.ThumbnailFile.RelativeOutputPath
	// }

	// return page, nil
	// 1. Base loading
	base := filepath.Base(path)

	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var matter domain.PageFrontmatter

	raw_content, err := frontmatter.Parse(file, &matter)
	if err != nil {
		return nil, err
	}

	// 1b. Thumbnail handling
	var thumbnail *domain.ContentFile = nil
	if matter.Thumbnail != nil {
		if ok, relPath, fullPath := util.CheckRelativeUrl(*matter.Thumbnail); ok {
			if util.Includes(filepath.Ext(relPath), domain.ImageMediaExtensions...) {
				thumbnailExt := filepath.Ext(*matter.Thumbnail)
				thumbnailPath := strings.TrimSuffix(*matter.Thumbnail, thumbnailExt) + ".thumb" + thumbnailExt

				thumbnail = &domain.ContentFile{
					Path:               filepath.Join(s.Path, fullPath),
					RelativePath:       filepath.Join(s.RelativePath, fullPath),
					RelativeOutputPath: filepath.Join(s.RelativePath, thumbnailPath),
					Kind:               domain.ContentFileTypeThumbnail,
				}
			} else {
				domain.L.Warnf(
					"In the page at %s,\nthe thumbnail is set to the value `%s`, but the path appears to point to a non-media file...\nAre you sure you are pointing to the right file?\nIgnoring value...",
					path,
					*matter.Thumbnail,
				)
			}
		} else {
			domain.L.Warnf(
				"In the page at %s,\nthe thumbnail is set to the value `%s`, but no file could be found at this path.\nIs it the right path?\nIgnoring value...",
				path,
				*matter.Thumbnail,
			)
		}
	}

	content := string(raw_content)

	// 2. Required metadata building
	format := Formats[filepath.Ext(base)]
	sort.Strings(matter.CWs)

	// 3. Page building
	page := &domain.Page{
		Section:       s,
		Path:          path,
		Content:       content,
		Draft:         matter.Draft,
		Format:        format,
		Template:      matter.Template,
		Date:          matter.Date,
		Matter:        matter,
		CWs:           matter.CWs,
		ThumbnailFile: thumbnail,
	}

	// 4. Rendering
	rendered, err := Parsers[page.Format].Parse(idx, page)
	if err != nil {
		return page, err
	}
	page.RenderedContent = rendered.Content
	page.Summary = rendered.Summary
	page.TOC = rendered.TOC
	page.LinksCount = rendered.Counts.Links
	page.MediasCount = rendered.Counts.Medias

	// 5. Derived metadata application
	baseWithoutExt := strings.TrimSuffix(base, filepath.Ext(base))
	if len(rendered.Title) != 0 {
		page.Title = rendered.Title
	} else if page.Matter.Title != "" {
		page.Title = page.Matter.Title
	} else {
		page.Title = baseWithoutExt
	}
	page.Slug = baseWithoutExt
	if page.Slug == "index" {
		page.Permalink = page.Section.Permalink
	} else {
		page.Permalink = filepath.Join(page.Section.Permalink, page.Slug+".html")
	}
	if page.ThumbnailFile != nil {
		page.Thumbnail = "/" + page.ThumbnailFile.RelativeOutputPath
	}

	return page, nil
	return nil, nil
}

// LoadFeed loads a feed's configuration
// func LoadFeed(s *domain.Section, path string) (*domain.Feed, error) {
// 	// 1. Base loading
// 	base := filepath.Base(path)

// 	file, err := os.Open(path)
// 	if err != nil {
// 		return nil, err
// 	}
// 	defer file.Close()
// 	data, _ := io.ReadAll(file)

// 2. Parsing
// var metadata struct {
// 	Title                string `validate:"required"`
// 	Description          string `validate:"required"`
// 	BaseURL              string `validate:"required,url" toml:"base_url"`
// 	AdvertizedURL        string `validate:"omitempty,url" toml:"advertized_url"`
// 	Slug                 string
// 	Extra                map[string]string
// 	PageSelectionPattern string `toml:"page_selection_pattern"`
// }

// if err = toml.Unmarshal(data, &metadata); err != nil {
// 	return nil, err
// }

// if err = validator.New().Struct(metadata); err != nil {
// 	return nil, err
// }

// baseURL, _ := url.Parse(metadata.BaseURL)
// advertizedURL := baseURL
// if metadata.AdvertizedURL != "" {
// 	advertizedURL, _ = url.Parse(metadata.AdvertizedURL)
// }

// 	// 3. Metadata building
// 	feed := &domain.Feed{
// 		Section:              s,
// 		Title:                metadata.Title,
// 		Description:          metadata.Description,
// 		BaseURL:              *baseURL,
// 		AdvertizedURL:        advertizedURL,
// 		Extra:                metadata.Extra,
// 		PageSelectionPattern: metadata.PageSelectionPattern,
// 	}

// 	// 4. Derived metadata application
// 	baseWithoutExt := strings.TrimSuffix(base, ".feed.toml")
// 	if metadata.Slug != "" {
// 		feed.Slug = metadata.Slug
// 	} else {
// 		feed.Slug = util.Slugify(baseWithoutExt)
// 	}
// 	feed.Permalink = filepath.Join(s.Permalink, feed.Slug+".xml")
// 	feed.Path = filepath.Join(filepath.Dir(path), feed.Slug+".xml")

// 	return feed, nil
// }

// func LoadRedirect(
// 	idx *domain.Index,
// 	fb *FsBuilder,


A render/render.go => render/render.go +39 -0
@@ 0,0 1,39 @@
package render

// ContentFormat is the source file's format.
type ContentFormat int

const (
	// ContentFormatGemini is for gemtext files (.gmi).
	ContentFormatGemini = iota
	// ContentFormatMarkdown is for markdown files (.md).
	ContentFormatMarkdown
	// ContentFormatHTML is for HTML files (.html)
	ContentFormatHTML
)

// Heading holds the metadata for a page / section heading.
type Heading struct {
	Level uint8
	Id    string
	Title string
}

// ParsingResult holds the result data bundle parsers should return
type ParsingResult struct {
	Content string
	Summary string
	TOC     []Heading
	Title   string
	Counts  struct {
		Links  uint // Links counts all the found links (including media links)
		Medias uint // Medias count all the found media links only
	}
}

// Parser exposes a parsing method taking a page, and returning a bundle of all rendered and derived data.
type Parser interface {
	// TODO: readapt
	// Parse(*domain.Index, *domain.Page) (ParsingResult, error)
	Parse(string) (ParsingResult, error)
}


A render/renderers.go => render/renderers.go +11 -0
@@ 0,0 1,11 @@
package render

var (
	Parsers = map[ContentFormat]Parser{}

	Formats = map[string]ContentFormat{
		"md":   ContentFormatMarkdown,
		"gmi":  ContentFormatGemini,
		"html": ContentFormatHTML,
	}
)


A transform/load_from_disk/load_page.go => transform/load_from_disk/load_page.go +98 -0
@@ 0,0 1,98 @@
package load_from_disk

import (
	"os"
	"time"

	"git.sr.ht/~artemis/cap/domain"
	"git.sr.ht/~artemis/cap/render"
	"git.sr.ht/~artemis/cap/util"
	"github.com/adrg/frontmatter"
)

// oh fuck this is gonna be such a horrendous hell
// TODO: todos left: 1. rendering, 2. thumbnail
func LoadPageFromDisk(f *domain.File) ([]domain.Leaf, error) {
	// 0. init + adding source leaf
	leaves := []domain.Leaf{f}

	// 1. loading file frontmatter
	file, err := os.Open(f.SourcePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var matter struct {
		Title     string     `yaml:"title"`
		CWs       []string   `yaml:"cw"`
		Draft     bool       `yaml:"draft"`
		Template  string     `yaml:"template"`
		Date      *time.Time `yaml:"date"`
		Thumbnail string     `yaml:"thumbnail"`
	}
	raw_content, err := frontmatter.Parse(file, &matter)
	if err != nil {
		return nil, err
	}
	content := string(raw_content)

	// 2. Render page content
	format := render.Formats[f.Permalink.Extension]
	parsingResult, err := render.Parsers[format].Parse(content)
	if err != nil {
		return nil, err
	}

	// 3. building page leaf
	title := matter.Title
	if title == "" {
		title = parsingResult.Title
	}
	if title == "" {
		title = f.Permalink.Slug
	}
	template := matter.Template
	if template == "" {
		if f.Permalink.Slug == "index" {
			template = "section"
		} else {
			template = "page"
		}
	}

	leaves = append(leaves, &domain.Page{
		Identity: domain.NewIdentity(
			f.Permalink.CloneWithExtension("html"),
			&f.ID,
		),

		Title:    title,
		Draft:    matter.Draft,
		Template: template,
		Date:     matter.Date,
		CWs:      matter.CWs,
		Summary:  parsingResult.Summary,
		TOC:      parsingResult.TOC,
		Content:  parsingResult.Content,
	})

	// 4. maybe building thumbnail leaf
	// TODO: probs need to add a ref to the thumbnail on the page leaf
	if matter.Thumbnail != "" {
		// TODO: rework CheckRelativeUrl to instead provide sensible generated paths
		if ok, _, _ := util.CheckRelativeUrl(matter.Thumbnail); ok {

			// leaves = append(leaves, &domain.Media{
			// 	Identity: domain.NewIdentity(
			// 		f.Permalink.CloneWithSlugAndExtension(, )
			// 	),
			// })
		} else {
			domain.L.Warnf("%s - no thumbnail could be found on-disk at %s", f.Permalink.Render(), matter.Thumbnail)
		}
	}

	// 5. done!
	return leaves, nil
}