~whereswaldon/forest-go

b5818c3bc3210c63ec78bb26988af444d9a8d3c7 — Chris Waldon 1 year, 1 month ago 960be2d
wip(store): initial work on subtree removal
6 files changed, 254 insertions(+), 23 deletions(-)

M store.go
M store/archive.go
M store/cache-store.go
M store/memory-store.go
A store/walker.go
A store/walker_test.go
M store.go => store.go +2 -0
@@ 16,4 16,6 @@ type Store interface {
	// Add inserts a node into the store. It is *not* an error to insert a node which is already
	// stored. Implementations must not return an error in this case.
	Add(Node) error

	RemoveSubtree(*fields.QualifiedHash) error
}

M store/archive.go => store/archive.go +22 -23
@@ 245,19 245,13 @@ func (a *Archive) AncestryOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash,
// in which the descendants are returned is undefined.
func (a *Archive) DescendantsOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	descendants := make([]*fields.QualifiedHash, 0)
	directChildren := []*fields.QualifiedHash{id}

	for len(directChildren) > 0 {
		target := directChildren[0]
		directChildren = directChildren[1:]
		children, err := a.Children(target)
		if err != nil {
			return nil, fmt.Errorf("failed looking up children of %s: %w", target, err)
		}
		for _, childID := range children {
			descendants = append(descendants, childID)
			directChildren = append(directChildren, childID)
		}
	err := Walk(a, id, func(id *fields.QualifiedHash) error {
		descendants = append(descendants, id)
		return nil
	})
	if err != nil {
		return nil, fmt.Errorf("failed traversing descendants: %w", err)
	}
	return descendants, nil
}


@@ 266,22 260,27 @@ func (a *Archive) DescendantsOf(id *fields.QualifiedHash) ([]*fields.QualifiedHa
// leaves is undefined.
func (a *Archive) LeavesOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	leaves := make([]*fields.QualifiedHash, 0)
	directChildren := []*fields.QualifiedHash{id}

	for len(directChildren) > 0 {
		target := directChildren[0]
		directChildren = directChildren[1:]
		children, err := a.Children(target)
	err := Walk(a, id, func(id *fields.QualifiedHash) error {
		children, err := a.Children(id)
		if err != nil {
			return nil, fmt.Errorf("failed looking up children of %s: %w", target, err)
			return fmt.Errorf("failed looking up children of %s: %w", id, err)
		}
		if len(children) == 0 {
			leaves = append(leaves, target)
			continue
		}
		for _, childID := range children {
			directChildren = append(directChildren, childID)
			leaves = append(leaves, id)
		}
		return nil
	})
	if err != nil {
		return nil, fmt.Errorf("failed traversing descendants: %w", err)
	}
	return leaves, nil
}

func (a *Archive) RemoveSubtree(id *fields.QualifiedHash) error {
	var err error
	a.executeAsync(func() {
		err = a.store.RemoveSubtree(id)
	})
	return err
}

M store/cache-store.go => store/cache-store.go +10 -0
@@ 109,3 109,13 @@ func (m *CacheStore) Children(id *fields.QualifiedHash) ([]*fields.QualifiedHash
func (m *CacheStore) Recent(nodeType fields.NodeType, quantity int) ([]forest.Node, error) {
	return m.Back.Recent(nodeType, quantity)
}

func (m *CacheStore) RemoveSubtree(id *fields.QualifiedHash) error {
	if err := m.Back.RemoveSubtree(id); err != nil {
		return fmt.Errorf("cachestore failed removing from backing store: %w", err)
	}
	if err := m.Cache.RemoveSubtree(id); err != nil {
		return fmt.Errorf("cachestore failed removing from cache: %w", err)
	}
	return nil
}

M store/memory-store.go => store/memory-store.go +31 -0
@@ 88,6 88,37 @@ func (m *MemoryStore) AddID(id string, node forest.Node) error {
	return nil
}

func (m *MemoryStore) RemoveSubtree(id *fields.QualifiedHash) error {
	children, err := m.Children(id)
	if err != nil {
		return fmt.Errorf("failed looking up children of %s: %w", id, err)
	}
	for _, child := range children {
		if err := m.RemoveSubtree(child); err != nil {
			return fmt.Errorf("failed removing children of %s: %w", child, err)
		}
	}
	child, _, err := m.Get(id)
	if err != nil {
		return fmt.Errorf("failed looking up child %s during removal: %w", id, err)
	}
	idString := id.String()
	parentIdString := child.ParentID().String()
	delete(m.Items, idString)
	siblings := m.ChildMap[parentIdString]
	for i := range siblings {
		if siblings[i] != idString {
			continue
		}
		for k := i + 1; k < len(siblings); k++ {
			siblings[i] = siblings[k]
		}
		m.ChildMap[parentIdString] = siblings[:len(siblings)-1]
		break
	}
	return nil
}

// Recent returns a slice of len `quantity` (or fewer) nodes of the given type.
// These nodes are the most recent (by creation time) nodes of that type known
// to the store.

A store/walker.go => store/walker.go +67 -0
@@ 0,0 1,67 @@
package store

import (
	"fmt"

	"git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
)

// Walk traverses the subtree rooted at start in a breadth-first fashion invoking the
// visitor function on each node id in the subtree. The traversal stops either
// when the visitor function returns non-nil or when the entire subtree
// rooted at start has been visited.
//
// If the visitor function returns an error, it will be returned wrapped and
// can be checked for using the errors.Is or errors.As standard library
// functions.
func Walk(s forest.Store, start *fields.QualifiedHash, visitor func(*fields.QualifiedHash) error) error {
	if s == nil {
		return fmt.Errorf("store cannot be nil")
	}
	if start == nil {
		return fmt.Errorf("start cannot be nil")
	}
	if visitor == nil {
		return fmt.Errorf("visitor cannot be nil")
	}

	childQueue := []*fields.QualifiedHash{start}
	var current *fields.QualifiedHash
	for len(childQueue) > 0 {
		current, childQueue = childQueue[0], childQueue[1:]
		err := visitor(current)
		if err != nil {
			return fmt.Errorf("visitor function errored on %s: %w", current, err)
		}
		children, err := s.Children(current)
		if err != nil {
			return fmt.Errorf("failed visiting children of %s: %w", current, err)
		}
		childQueue = append(childQueue, children...)
	}
	return nil
}

// WalkNodes traverses the subtree rooted at start in a breadth-first fashion invoking the
// visitor function on each node in the subtree.
func WalkNodes(s forest.Store, start forest.Node, visitor func(forest.Node) error) (err error) {
	defer func() {
		if err != nil {
			err = fmt.Errorf("failed walking nodes: %w", err)
		}
	}()
	var (
		node forest.Node
		has  bool
	)
	return Walk(s, start.ID(), func(id *fields.QualifiedHash) error {
		node, has, err = s.Get(id)
		if err != nil {
			return err
		} else if !has {
			return fmt.Errorf("tried to visit nonexistent node: %s", id)
		}
		return visitor(node)
	})
}

A store/walker_test.go => store/walker_test.go +122 -0
@@ 0,0 1,122 @@
package store_test

import (
	"errors"
	"fmt"
	"math/rand"
	"sort"
	"strconv"
	"strings"
	"testing"

	"git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
	"git.sr.ht/~whereswaldon/forest-go/store"
	"git.sr.ht/~whereswaldon/forest-go/testutil"
)

func prep(t *testing.T) (s forest.Store, root *fields.QualifiedHash, ids []*fields.QualifiedHash) {
	id, signer, community, reply := testutil.MakeReplyOrSkip(t)
	builder := forest.Builder{
		User:   id,
		Signer: signer,
	}

	s = store.NewMemoryStore()
	s.Add(id)

	nodes := []forest.Node{community, reply}
	for i := 0; i < 10; i++ {
		parent := nodes[rand.Intn(len(nodes))]
		n, err := builder.NewReply(parent, strconv.Itoa(i), []byte{})
		if err != nil {
			t.Errorf("failed generating test node: %v", err)
		}
		nodes = append(nodes, n)
	}
	ids = []*fields.QualifiedHash{}
	for _, node := range nodes {
		ids = append(ids, node.ID())
		s.Add(node)
	}

	return s, community.ID(), ids
}

func TestWalk(t *testing.T) {
	s, root, ids := prep(t)

	reachedIDs := []*fields.QualifiedHash{}
	store.Walk(s, root, func(id *fields.QualifiedHash) error {
		reachedIDs = append(reachedIDs, id)
		return nil
	})

	sortIds(ids)
	sortIds(reachedIDs)
	if !sameIds(ids, reachedIDs) {
		t.Errorf("failed to reach all nodes in walk: expected %v, got %v", ids, reachedIDs)
	}
	t.Log(ids)
	t.Log(reachedIDs)
}

func TestWalkTerminate(t *testing.T) {
	s, root, _ := prep(t)

	count := 0
	stop := fmt.Errorf("stop")
	reachedIDs := []*fields.QualifiedHash{}
	if err := store.Walk(s, root, func(id *fields.QualifiedHash) error {
		if count >= 5 {
			return stop
		}
		count++
		reachedIDs = append(reachedIDs, id)
		return nil
	}); err != nil {
		if !errors.Is(err, stop) {
			t.Errorf("should have returned wrapped stop error")
		}
	}

	if len(reachedIDs) != count {
		t.Errorf("visited an unexpected number of nodes, expected %d, visited %d", count, len(reachedIDs))
	}
}

func sortIds(ids []*fields.QualifiedHash) {
	sort.Slice(ids, func(i, j int) bool {
		return strings.Compare(ids[i].String(), ids[j].String()) < 0
	})
}

func sameIds(a, b []*fields.QualifiedHash) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if !a[i].Equals(b[i]) {
			return false
		}
	}
	return true
}

func TestWalkNils(t *testing.T) {
	_, _, community, _ := testutil.MakeReplyOrSkip(t)
	s := store.NewMemoryStore()
	if err := store.Walk(nil, community.ID(), func(*fields.QualifiedHash) error {
		return nil
	}); err == nil {
		t.Errorf("Walk should error with nil store")
	}
	if err := store.Walk(s, nil, func(*fields.QualifiedHash) error {
		return nil
	}); err == nil {
		t.Errorf("Walk should error with root")
	}
	if err := store.Walk(s, community.ID(), nil); err == nil {
		t.Errorf("Walk should error with nil visitor")
	}
}