~whereswaldon/forest-go

4089ee7e5d938ee4ec4805d98d4e61f014c58fb7 — Chris Waldon 1 year, 4 months ago 0859340
feat: reorganize Store implementations into new package
9 files changed, 581 insertions(+), 458 deletions(-)

D archive/archive.go
D archive/archive_test.go
M grove/grove.go
M store.go
A store/archive.go
A store/cache-store.go
A store/extended-store.go
A store/memory-store.go
R store_test.go => store/store_test.go
D archive/archive.go => archive/archive.go +0 -92
@@ 1,92 0,0 @@
/*
Package archive defines a helpful wrapper type to augment the store interface
with higher-level methods for querying ancestry and descendants of nodes in
the store.
*/
package archive

import (
	"fmt"

	"git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
)

// Archive extends a store with methods high-level structural queries.
type Archive struct {
	forest.Store
}

// New constructs a new Archive wrapping the given store
func New(store forest.Store) *Archive {
	return &Archive{store}
}

// AncestryOf returns the IDs of all known ancestors of the node with the given `id`. The ancestors are
// returned sorted by descending depth, so the root of the ancestry tree is the final node in the slice.
func (a *Archive) AncestryOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	node, present, err := a.Store.Get(id)
	if err != nil {
		return nil, fmt.Errorf("failed looking up %s: %w", id, err)
	} else if !present {
		return []*fields.QualifiedHash{}, nil
	}
	ancestors := make([]*fields.QualifiedHash, 0, node.TreeDepth())
	next := node.ParentID()
	for !next.Equals(fields.NullHash()) {
		parent, present, err := a.Store.Get(next)
		if err != nil {
			return nil, fmt.Errorf("failed looking up ancestor %s: %w", next, err)
		} else if !present {
			return ancestors, nil
		}
		ancestors = append(ancestors, next)
		next = parent.ParentID()
	}
	return ancestors, nil
}

// DescendantsOf returns the IDs of all known descendants of the node with the given `id`. The order
// in which the descendants are returned is undefined.
func (v *Archive) DescendantsOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	descendants := make([]*fields.QualifiedHash, 0)
	directChildren := []*fields.QualifiedHash{id}

	for len(directChildren) > 0 {
		target := directChildren[0]
		directChildren = directChildren[1:]
		children, err := v.Children(target)
		if err != nil {
			return nil, fmt.Errorf("failed looking up children of %s: %w", target, err)
		}
		for _, childID := range children {
			descendants = append(descendants, childID)
			directChildren = append(directChildren, childID)
		}
	}
	return descendants, nil
}

// LeavesOf returns the leaf nodes of the tree rooted at `id`. The order of the returned
// leaves is undefined.
func (v *Archive) LeavesOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	leaves := make([]*fields.QualifiedHash, 0)
	directChildren := []*fields.QualifiedHash{id}

	for len(directChildren) > 0 {
		target := directChildren[0]
		directChildren = directChildren[1:]
		children, err := v.Children(target)
		if err != nil {
			return nil, fmt.Errorf("failed looking up children of %s: %w", target, err)
		}
		if len(children) == 0 {
			leaves = append(leaves, target)
			continue
		}
		for _, childID := range children {
			directChildren = append(directChildren, childID)
		}
	}
	return leaves, nil
}

D archive/archive_test.go => archive/archive_test.go +0 -130
@@ 1,130 0,0 @@
package archive_test

import (
	"testing"

	"git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/archive"
	"git.sr.ht/~whereswaldon/forest-go/fields"
	"git.sr.ht/~whereswaldon/forest-go/testkeys"
)

type setup struct {
	identity, community, r1, r2, r3a, r3b forest.Node
}

func testStore(t *testing.T) (forest.Store, setup) {
	store := forest.NewMemoryStore()
	setup := setup{}
	signer := testkeys.Signer(t, testkeys.PrivKey1)
	identity, err := forest.NewIdentity(signer, "archivetest", []byte{})
	if err != nil {
		t.Fatalf("failed making test identity: %v", err)
	}
	store.Add(identity)
	setup.identity = identity
	builder := forest.As(identity, signer)
	community, err := builder.NewCommunity("archivetest1", []byte{})
	if err != nil {
		t.Fatalf("failed making test community: %v", err)
	}
	store.Add(community)
	setup.community = community
	setup.r1, err = builder.NewReply(community, "r1", []byte{})
	if err != nil {
		t.Fatalf("failing creating reply: %v", err)
	}
	store.Add(setup.r1)
	setup.r2, err = builder.NewReply(setup.r1, "r2", []byte{})
	if err != nil {
		t.Fatalf("failing creating reply: %v", err)
	}
	store.Add(setup.r2)
	setup.r3a, err = builder.NewReply(setup.r2, "r3a", []byte{})
	if err != nil {
		t.Fatalf("failing creating reply: %v", err)
	}
	store.Add(setup.r3a)
	setup.r3b, err = builder.NewReply(setup.r2, "r3b", []byte{})
	if err != nil {
		t.Fatalf("failing creating reply: %v", err)
	}
	store.Add(setup.r3b)
	return store, setup
}

func TestAncestryOf(t *testing.T) {
	store, setup := testStore(t)
	arch := archive.New(store)
	ancestry, err := arch.AncestryOf(setup.r3a.ID())
	if err != nil {
		t.Fatalf("failed fetching ancestry of %s: %v", setup.r3a.ID(), err)
	}
	r3aAncestors := 3
	if len(ancestry) != r3aAncestors {
		t.Fatalf("expected %d nodes of history for %s, got %d", r3aAncestors, setup.r3a.ID(), len(ancestry))
	}
	switch {
	case !ancestry[0].Equals(setup.r2.ID()):
		fallthrough
	case !ancestry[1].Equals(setup.r1.ID()):
		fallthrough
	case !ancestry[2].Equals(setup.community.ID()):
		t.Fatalf("incorrect ancestry: %v", ancestry)
	}
}

func contains(ids []*fields.QualifiedHash, id *fields.QualifiedHash) bool {
	for _, element := range ids {
		if element.Equals(id) {
			return true
		}
	}
	return false
}

func TestDescendantsOf(t *testing.T) {
	store, setup := testStore(t)
	arch := archive.New(store)
	descendants, err := arch.DescendantsOf(setup.r1.ID())
	if err != nil {
		t.Fatalf("failed fetching descendants of %s: %v", setup.r1.ID(), err)
	}
	switch {
	case contains(descendants, setup.community.ID()):
		t.Fatalf("community is not descendant of conversation")
	case contains(descendants, setup.identity.ID()):
		t.Fatalf("identity is not descendant of conversation")
	case contains(descendants, setup.r1.ID()):
		t.Fatalf("r1 is not a descendant of itself")
	case !contains(descendants, setup.r2.ID()):
		fallthrough
	case !contains(descendants, setup.r3a.ID()):
		fallthrough
	case !contains(descendants, setup.r3b.ID()):
		t.Fatalf("descendants is missing nodes")
	}
}

func TestLeavesOf(t *testing.T) {
	store, setup := testStore(t)
	arch := archive.New(store)
	leaves, err := arch.LeavesOf(setup.r1.ID())
	if err != nil {
		t.Fatalf("failed fetching leaves of %s: %v", setup.r1.ID(), err)
	}
	switch {
	case contains(leaves, setup.community.ID()):
		t.Fatalf("community is not descendant of conversation")
	case contains(leaves, setup.identity.ID()):
		t.Fatalf("identity is not descendant of conversation")
	case contains(leaves, setup.r1.ID()):
		t.Fatalf("r1 is not a leaf of itself")
	case contains(leaves, setup.r2.ID()):
		t.Fatalf("r2 is not a leaf of r1")
	case !contains(leaves, setup.r3a.ID()):
		fallthrough
	case !contains(leaves, setup.r3b.ID()):
		t.Fatalf("leaves is missing nodes")
	}
}

M grove/grove.go => grove/grove.go +3 -2
@@ 20,6 20,7 @@ import (

	"git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
	"git.sr.ht/~whereswaldon/forest-go/store"
)

// File represents a type that supports file-like operations. *os.File


@@ 79,7 80,7 @@ func (r RelativeFS) OpenFile(path string, flag int, perm os.FileMode) (File, err
// event of a disk modification is to call RebuildChildCache().
type Grove struct {
	FS
	NodeCache *forest.MemoryStore
	NodeCache *store.MemoryStore
	*ChildCache
}



@@ 97,7 98,7 @@ func NewWithFS(fs FS) (*Grove, error) {
	}
	return &Grove{
		FS:         fs,
		NodeCache:  forest.NewMemoryStore(),
		NodeCache:  store.NewMemoryStore(),
		ChildCache: NewChildCache(),
	}, nil
}

M store.go => store.go +0 -223
@@ 1,9 1,6 @@
package forest

import (
	"fmt"
	"sort"

	"git.sr.ht/~whereswaldon/forest-go/fields"
)



@@ 20,223 17,3 @@ type Store interface {
	// stored. Implementations must not return an error in this case.
	Add(Node) error
}

type MemoryStore struct {
	Items    map[string]Node
	ChildMap map[string][]string
}

var _ Store = &MemoryStore{}

func NewMemoryStore() *MemoryStore {
	return &MemoryStore{
		Items:    make(map[string]Node),
		ChildMap: make(map[string][]string),
	}
}

func (m *MemoryStore) CopyInto(other Store) error {
	for _, node := range m.Items {
		if err := other.Add(node); err != nil {
			return err
		}
	}
	return nil
}

func (m *MemoryStore) Get(id *fields.QualifiedHash) (Node, bool, error) {
	return m.GetID(id.String())
}

func (m *MemoryStore) GetIdentity(id *fields.QualifiedHash) (Node, bool, error) {
	return m.Get(id)
}

func (m *MemoryStore) GetCommunity(id *fields.QualifiedHash) (Node, bool, error) {
	return m.Get(id)
}

func (m *MemoryStore) GetConversation(communityID, conversationID *fields.QualifiedHash) (Node, bool, error) {
	return m.Get(conversationID)
}

func (m *MemoryStore) GetReply(communityID, conversationID, replyID *fields.QualifiedHash) (Node, bool, error) {
	return m.Get(replyID)
}

func (m *MemoryStore) GetID(id string) (Node, bool, error) {
	item, has := m.Items[id]
	return item, has, nil
}

func (m *MemoryStore) Children(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	idString := id.String()
	children, any := m.ChildMap[idString]
	if !any {
		return []*fields.QualifiedHash{}, nil
	}
	childIDs := make([]*fields.QualifiedHash, len(children))
	for i, childStr := range children {
		childIDs[i] = &fields.QualifiedHash{}
		if err := childIDs[i].UnmarshalText([]byte(childStr)); err != nil {
			return nil, fmt.Errorf("failed to transform key back into node id: %w", err)
		}
	}
	return childIDs, nil
}

func (m *MemoryStore) Add(node Node) error {
	id := node.ID().String()
	return m.AddID(id, node)
}

func (m *MemoryStore) AddID(id string, node Node) error {
	// safe to ignore error because we know it can't happen
	if _, has, _ := m.GetID(id); has {
		return nil
	}
	m.Items[id] = node
	parentID := node.ParentID().String()
	m.ChildMap[parentID] = append(m.ChildMap[parentID], id)
	return nil
}

// Recent returns a slice of len `quantity` (or fewer) nodes of the given type.
// These nodes are the most recent (by creation time) nodes of that type known
// to the store.
func (m *MemoryStore) Recent(nodeType fields.NodeType, quantity int) ([]Node, error) {
	// highly inefficient implementation, but it should work for now
	candidates := make([]Node, 0, quantity)
	for _, node := range m.Items {
		switch n := node.(type) {
		case *Identity:
			if nodeType == fields.NodeTypeIdentity {
				candidates = append(candidates, n)
				sort.SliceStable(candidates, func(i, j int) bool {
					return candidates[i].(*Identity).Created > candidates[j].(*Identity).Created
				})
			}
		case *Community:
			if nodeType == fields.NodeTypeCommunity {
				candidates = append(candidates, n)
				sort.SliceStable(candidates, func(i, j int) bool {
					return candidates[i].(*Community).Created > candidates[j].(*Community).Created
				})
			}
		case *Reply:
			if nodeType == fields.NodeTypeReply {
				candidates = append(candidates, n)
				sort.SliceStable(candidates, func(i, j int) bool {
					return candidates[i].(*Reply).Created > candidates[j].(*Reply).Created
				})
			}
		}
	}
	if len(candidates) > quantity {
		candidates = candidates[:quantity]
	}
	return candidates, nil
}

// CacheStore combines two other stores into one logical store. It is
// useful when store implementations have different performance
// characteristics and one is dramatically faster than the other. Once
// a CacheStore is created, the individual stores within it should not
// be directly modified.
type CacheStore struct {
	Cache, Back Store
}

var _ Store = &CacheStore{}

// NewCacheStore creates a single logical store from the given two stores.
// All items from `cache` are automatically copied into `base` during
// the construction of the CacheStore, and from then on (assuming
// neither store is modified directly outside of CacheStore) all elements
// added are guaranteed to be added to `base`. It is recommended to use
// fast in-memory implementations as the `cache` layer and disk or
// network-based implementations as the `base` layer.
func NewCacheStore(cache, back Store) (*CacheStore, error) {
	if err := cache.CopyInto(back); err != nil {
		return nil, err
	}
	return &CacheStore{cache, back}, nil
}

// Get returns the requested node if it is present in either the Cache or the Back Store.
// If the cache is missed by the backing store is hit, the node will automatically be
// added to the cache.
func (m *CacheStore) Get(id *fields.QualifiedHash) (Node, bool, error) {
	return m.getUsingFuncs(id, m.Cache.Get, m.Back.Get)
}

func (m *CacheStore) CopyInto(other Store) error {
	return m.Back.CopyInto(other)
}

// Add inserts the given node into both stores of the CacheStore
func (m *CacheStore) Add(node Node) error {
	if err := m.Back.Add(node); err != nil {
		return err
	}
	if err := m.Cache.Add(node); err != nil {
		return err
	}
	return nil
}

func (m *CacheStore) getUsingFuncs(id *fields.QualifiedHash, getter1, getter2 func(*fields.QualifiedHash) (Node, bool, error)) (Node, bool, error) {
	cacheNode, inCache, err := getter1(id)
	if err != nil {
		return nil, false, fmt.Errorf("failed fetching id from cache: %w", err)
	}
	if inCache {
		return cacheNode, inCache, err
	}
	backNode, inBackingStore, err := getter2(id)
	if err != nil {
		return nil, false, fmt.Errorf("failed fetching id from cache: %w", err)
	}
	if inBackingStore {
		if err := m.Cache.Add(backNode); err != nil {
			return nil, false, fmt.Errorf("failed to up-propagate node into cache: %w", err)
		}
	}
	return backNode, inBackingStore, err
}

func (m *CacheStore) GetIdentity(id *fields.QualifiedHash) (Node, bool, error) {
	return m.getUsingFuncs(id, m.Cache.GetIdentity, m.Back.GetIdentity)
}

func (m *CacheStore) GetCommunity(id *fields.QualifiedHash) (Node, bool, error) {
	return m.getUsingFuncs(id, m.Cache.GetCommunity, m.Back.GetCommunity)
}

func (m *CacheStore) GetConversation(communityID, conversationID *fields.QualifiedHash) (Node, bool, error) {
	return m.getUsingFuncs(communityID, // this id is irrelevant
		func(*fields.QualifiedHash) (Node, bool, error) {
			return m.Cache.GetConversation(communityID, conversationID)
		},
		func(*fields.QualifiedHash) (Node, bool, error) {
			return m.Back.GetConversation(communityID, conversationID)
		})
}

func (m *CacheStore) GetReply(communityID, conversationID, replyID *fields.QualifiedHash) (Node, bool, error) {
	return m.getUsingFuncs(communityID, // this id is irrelevant
		func(*fields.QualifiedHash) (Node, bool, error) {
			return m.Cache.GetReply(communityID, conversationID, replyID)
		},
		func(*fields.QualifiedHash) (Node, bool, error) {
			return m.Back.GetReply(communityID, conversationID, replyID)
		})
}

func (m *CacheStore) Children(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	return m.Back.Children(id)
}

func (m *CacheStore) Recent(nodeType fields.NodeType, quantity int) ([]Node, error) {
	return m.Back.Recent(nodeType, quantity)
}

A store/archive.go => store/archive.go +310 -0
@@ 0,0 1,310 @@
package store

import (
	"fmt"

	"git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
)

// Subscription is an identifier for a particular handler function within
// a SubscriberStore. It can be provided to delete a handler function or to
// suppress notifications to the corresponding handler.
type Subscription uint

// the zero subscription is never used
const neverAssigned = 0
const firstSubscription = 1

// Archive is a wrapper type that extends the forest.Store interface
// with the observer pattern. Code can subscribe for updates each time a
// node is inserted into the store using Add or AddAs.
type Archive struct {
	store                                 forest.Store
	requests                              chan func()
	nextSubscriberKey                     Subscription
	postAddSubscribers, preAddSubscribers map[Subscription]func(forest.Node)
}

var _ ExtendedStore = &Archive{}

// NewArchive creates a thread-safe storage structure for
// forest nodes by wrapping an existing store implementation
func NewArchive(store forest.Store) *Archive {
	m := &Archive{
		store:              store,
		requests:           make(chan func()),
		nextSubscriberKey:  firstSubscription,
		postAddSubscribers: make(map[Subscription]func(forest.Node)),
		preAddSubscribers:  make(map[Subscription]func(forest.Node)),
	}
	go func() {
		for function := range m.requests {
			function()
		}
	}()
	return m
}

// SubscribeToNewMessages establishes the given function as a handler to be
// invoked on each node added to the store. The returned subscription ID
// can be used to unsubscribe later, as well as to supress notifications
// with AddAs().
//
// Handler functions are invoked synchronously on the same goroutine that invokes
// Add() or AddAs(), and should not block. If long-running code is needed in a
// handler, launch a new goroutine.
func (m *Archive) SubscribeToNewMessages(handler func(n forest.Node)) (subscriptionID Subscription) {
	return m.subscribeInMap(m.postAddSubscribers, handler)
}

// PresubscribeToNewMessages establishes the given function as a handler to be
// invoked on each node added to the store. The returned subscription ID
// can be used to unsubscribe later, as well as to supress notifications
// with AddAs(). The handler function will be invoked *before* nodes are
// inserted into the store instead of after (like a normal Subscribe).
//
// Handler functions are invoked synchronously on the same goroutine that invokes
// Add() or AddAs(), and should not block. If long-running code is needed in a
// handler, launch a new goroutine.
func (m *Archive) PresubscribeToNewMessages(handler func(n forest.Node)) (subscriptionID Subscription) {
	return m.subscribeInMap(m.preAddSubscribers, handler)
}

func (m *Archive) subscribeInMap(targetMap map[Subscription]func(forest.Node), handler func(n forest.Node)) (subscriptionID Subscription) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		subscriptionID = m.nextSubscriberKey
		m.nextSubscriberKey++
		// handler unsigned overflow
		// TODO: ensure subscription reuse can't occur
		if m.nextSubscriberKey == neverAssigned {
			m.nextSubscriberKey = firstSubscription
		}
		targetMap[subscriptionID] = handler
	}
	<-done
	return
}

// UnsubscribeToNewMessages removes the handler for a given subscription from
// the store.
func (m *Archive) UnsubscribeToNewMessages(subscriptionID Subscription) {
	m.unsubscribeInMap(m.postAddSubscribers, subscriptionID)
}

// UnpresubscribeToNewMessages removes the handler for a given subscription from
// the store.
func (m *Archive) UnpresubscribeToNewMessages(subscriptionID Subscription) {
	m.unsubscribeInMap(m.preAddSubscribers, subscriptionID)
}

func (m *Archive) unsubscribeInMap(targetMap map[Subscription]func(forest.Node), subscriptionID Subscription) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		if _, subscribed := targetMap[subscriptionID]; subscribed {
			delete(targetMap, subscriptionID)
		}
	}
	<-done
	return
}

func (m *Archive) CopyInto(s forest.Store) (err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		err = m.store.CopyInto(s)
	}
	<-done
	return
}

func (m *Archive) Get(id *fields.QualifiedHash) (node forest.Node, present bool, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		node, present, err = m.store.Get(id)
	}
	<-done
	return
}

func (m *Archive) GetIdentity(id *fields.QualifiedHash) (node forest.Node, present bool, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		node, present, err = m.store.GetIdentity(id)
	}
	<-done
	return
}

func (m *Archive) GetCommunity(id *fields.QualifiedHash) (node forest.Node, present bool, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		node, present, err = m.store.GetCommunity(id)
	}
	<-done
	return
}

func (m *Archive) GetConversation(communityID, conversationID *fields.QualifiedHash) (node forest.Node, present bool, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		node, present, err = m.store.GetConversation(communityID, conversationID)
	}
	<-done
	return
}

func (m *Archive) GetReply(communityID, conversationID, replyID *fields.QualifiedHash) (node forest.Node, present bool, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		node, present, err = m.store.GetReply(communityID, conversationID, replyID)
	}
	<-done
	return
}

func (m *Archive) Children(id *fields.QualifiedHash) (ids []*fields.QualifiedHash, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		ids, err = m.store.Children(id)
	}
	<-done
	return
}

func (m *Archive) Recent(nodeType fields.NodeType, quantity int) (nodes []forest.Node, err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		nodes, err = m.store.Recent(nodeType, quantity)
	}
	<-done
	return
}

// Add inserts a node into the underlying store. Importantly, this will send a notification
// of a new node to *all* subscribers. If the calling code is a subscriber, it will still
// be notified of the new node. To supress this, use AddAs() instead.
func (m *Archive) Add(node forest.Node) (err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		m.notifySubscribed(m.preAddSubscribers, node, neverAssigned)
		if err = m.store.Add(node); err == nil {
			m.notifySubscribed(m.postAddSubscribers, node, neverAssigned)
		}
	}
	<-done
	return
}

// AddAs allows adding a node to the underlying store without being notified
// of it as a new node. The addedByID (subscription id returned from SubscribeToNewMessages)
// will not be notified of the new nodes, but all other subscribers will be.
func (m *Archive) AddAs(node forest.Node, addedByID Subscription) (err error) {
	done := make(chan struct{})
	m.requests <- func() {
		defer close(done)
		m.notifySubscribed(m.preAddSubscribers, node, addedByID)
		if err = m.store.Add(node); err == nil {
			m.notifySubscribed(m.postAddSubscribers, node, addedByID)
		}
	}
	<-done
	return
}

// notifySubscribed runs all of the subscription handlers in new goroutines with
// the provided node as input to each handler.
func (m *Archive) notifySubscribed(targetMap map[Subscription]func(forest.Node), node forest.Node, ignore Subscription) {
	for subscriptionID, handler := range targetMap {
		if subscriptionID != ignore {
			handler(node)
		}
	}
}

// Shut down the worker gorountine that powers this store. Subsequent
// calls to methods on this MessageStore have undefined behavior
func (m *Archive) Destroy() {
	close(m.requests)
}

// AncestryOf returns the IDs of all known ancestors of the node with the given `id`. The ancestors are
// returned sorted by descending depth, so the root of the ancestry tree is the final node in the slice.
func (a *Archive) AncestryOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	node, present, err := a.Get(id)
	if err != nil {
		return nil, fmt.Errorf("failed looking up %s: %w", id, err)
	} else if !present {
		return []*fields.QualifiedHash{}, nil
	}
	ancestors := make([]*fields.QualifiedHash, 0, node.TreeDepth())
	next := node.ParentID()
	for !next.Equals(fields.NullHash()) {
		parent, present, err := a.Get(next)
		if err != nil {
			return nil, fmt.Errorf("failed looking up ancestor %s: %w", next, err)
		} else if !present {
			return ancestors, nil
		}
		ancestors = append(ancestors, next)
		next = parent.ParentID()
	}
	return ancestors, nil
}

// DescendantsOf returns the IDs of all known descendants of the node with the given `id`. The order
// in which the descendants are returned is undefined.
func (a *Archive) DescendantsOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	descendants := make([]*fields.QualifiedHash, 0)
	directChildren := []*fields.QualifiedHash{id}

	for len(directChildren) > 0 {
		target := directChildren[0]
		directChildren = directChildren[1:]
		children, err := a.Children(target)
		if err != nil {
			return nil, fmt.Errorf("failed looking up children of %s: %w", target, err)
		}
		for _, childID := range children {
			descendants = append(descendants, childID)
			directChildren = append(directChildren, childID)
		}
	}
	return descendants, nil
}

// LeavesOf returns the leaf nodes of the tree rooted at `id`. The order of the returned
// leaves is undefined.
func (a *Archive) LeavesOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	leaves := make([]*fields.QualifiedHash, 0)
	directChildren := []*fields.QualifiedHash{id}

	for len(directChildren) > 0 {
		target := directChildren[0]
		directChildren = directChildren[1:]
		children, err := a.Children(target)
		if err != nil {
			return nil, fmt.Errorf("failed looking up children of %s: %w", target, err)
		}
		if len(children) == 0 {
			leaves = append(leaves, target)
			continue
		}
		for _, childID := range children {
			directChildren = append(directChildren, childID)
		}
	}
	return leaves, nil
}

A store/cache-store.go => store/cache-store.go +111 -0
@@ 0,0 1,111 @@
package store

import (
	"fmt"

	forest "git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
)

// CacheStore combines two other stores into one logical store. It is
// useful when store implementations have different performance
// characteristics and one is dramatically faster than the other. Once
// a CacheStore is created, the individual stores within it should not
// be directly modified.
type CacheStore struct {
	Cache, Back forest.Store
}

var _ forest.Store = &CacheStore{}

// NewCacheStore creates a single logical store from the given two stores.
// All items from `cache` are automatically copied into `base` during
// the construction of the CacheStore, and from then on (assuming
// neither store is modified directly outside of CacheStore) all elements
// added are guaranteed to be added to `base`. It is recommended to use
// fast in-memory implementations as the `cache` layer and disk or
// network-based implementations as the `base` layer.
func NewCacheStore(cache, back forest.Store) (*CacheStore, error) {
	if err := cache.CopyInto(back); err != nil {
		return nil, err
	}
	return &CacheStore{cache, back}, nil
}

// Get returns the requested node if it is present in either the Cache or the Back Store.
// If the cache is missed by the backing store is hit, the node will automatically be
// added to the cache.
func (m *CacheStore) Get(id *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.getUsingFuncs(id, m.Cache.Get, m.Back.Get)
}

func (m *CacheStore) CopyInto(other forest.Store) error {
	return m.Back.CopyInto(other)
}

// Add inserts the given node into both stores of the CacheStore
func (m *CacheStore) Add(node forest.Node) error {
	if err := m.Back.Add(node); err != nil {
		return err
	}
	if err := m.Cache.Add(node); err != nil {
		return err
	}
	return nil
}

func (m *CacheStore) getUsingFuncs(id *fields.QualifiedHash, getter1, getter2 func(*fields.QualifiedHash) (forest.Node, bool, error)) (forest.Node, bool, error) {
	cacheNode, inCache, err := getter1(id)
	if err != nil {
		return nil, false, fmt.Errorf("failed fetching id from cache: %w", err)
	}
	if inCache {
		return cacheNode, inCache, err
	}
	backNode, inBackingStore, err := getter2(id)
	if err != nil {
		return nil, false, fmt.Errorf("failed fetching id from cache: %w", err)
	}
	if inBackingStore {
		if err := m.Cache.Add(backNode); err != nil {
			return nil, false, fmt.Errorf("failed to up-propagate node into cache: %w", err)
		}
	}
	return backNode, inBackingStore, err
}

func (m *CacheStore) GetIdentity(id *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.getUsingFuncs(id, m.Cache.GetIdentity, m.Back.GetIdentity)
}

func (m *CacheStore) GetCommunity(id *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.getUsingFuncs(id, m.Cache.GetCommunity, m.Back.GetCommunity)
}

func (m *CacheStore) GetConversation(communityID, conversationID *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.getUsingFuncs(communityID, // this id is irrelevant
		func(*fields.QualifiedHash) (forest.Node, bool, error) {
			return m.Cache.GetConversation(communityID, conversationID)
		},
		func(*fields.QualifiedHash) (forest.Node, bool, error) {
			return m.Back.GetConversation(communityID, conversationID)
		})
}

func (m *CacheStore) GetReply(communityID, conversationID, replyID *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.getUsingFuncs(communityID, // this id is irrelevant
		func(*fields.QualifiedHash) (forest.Node, bool, error) {
			return m.Cache.GetReply(communityID, conversationID, replyID)
		},
		func(*fields.QualifiedHash) (forest.Node, bool, error) {
			return m.Back.GetReply(communityID, conversationID, replyID)
		})
}

func (m *CacheStore) Children(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	return m.Back.Children(id)
}

func (m *CacheStore) Recent(nodeType fields.NodeType, quantity int) ([]forest.Node, error) {
	return m.Back.Recent(nodeType, quantity)
}

A store/extended-store.go => store/extended-store.go +19 -0
@@ 0,0 1,19 @@
package store

import (
	forest "git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
)

// ExtendedStore provides a superset of the functionality of the Store interface,
// implementing methods for subscribing to changes and querying higher-level
// structural information like ancestry and descendants.
type ExtendedStore interface {
	forest.Store
	SubscribeToNewMessages(handler func(n forest.Node)) Subscription
	UnsubscribeToNewMessages(Subscription)
	AddAs(forest.Node, Subscription) (err error)
	AncestryOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error)
	DescendantsOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error)
	LeavesOf(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error)
}

A store/memory-store.go => store/memory-store.go +126 -0
@@ 0,0 1,126 @@
package store

import (
	"fmt"
	"sort"

	forest "git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
)

type MemoryStore struct {
	Items    map[string]forest.Node
	ChildMap map[string][]string
}

var _ forest.Store = &MemoryStore{}

func NewMemoryStore() *MemoryStore {
	return &MemoryStore{
		Items:    make(map[string]forest.Node),
		ChildMap: make(map[string][]string),
	}
}

func (m *MemoryStore) CopyInto(other forest.Store) error {
	for _, node := range m.Items {
		if err := other.Add(node); err != nil {
			return err
		}
	}
	return nil
}

func (m *MemoryStore) Get(id *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.GetID(id.String())
}

func (m *MemoryStore) GetIdentity(id *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.Get(id)
}

func (m *MemoryStore) GetCommunity(id *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.Get(id)
}

func (m *MemoryStore) GetConversation(communityID, conversationID *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.Get(conversationID)
}

func (m *MemoryStore) GetReply(communityID, conversationID, replyID *fields.QualifiedHash) (forest.Node, bool, error) {
	return m.Get(replyID)
}

func (m *MemoryStore) GetID(id string) (forest.Node, bool, error) {
	item, has := m.Items[id]
	return item, has, nil
}

func (m *MemoryStore) Children(id *fields.QualifiedHash) ([]*fields.QualifiedHash, error) {
	idString := id.String()
	children, any := m.ChildMap[idString]
	if !any {
		return []*fields.QualifiedHash{}, nil
	}
	childIDs := make([]*fields.QualifiedHash, len(children))
	for i, childStr := range children {
		childIDs[i] = &fields.QualifiedHash{}
		if err := childIDs[i].UnmarshalText([]byte(childStr)); err != nil {
			return nil, fmt.Errorf("failed to transform key back into node id: %w", err)
		}
	}
	return childIDs, nil
}

func (m *MemoryStore) Add(node forest.Node) error {
	id := node.ID().String()
	return m.AddID(id, node)
}

func (m *MemoryStore) AddID(id string, node forest.Node) error {
	// safe to ignore error because we know it can't happen
	if _, has, _ := m.GetID(id); has {
		return nil
	}
	m.Items[id] = node
	parentID := node.ParentID().String()
	m.ChildMap[parentID] = append(m.ChildMap[parentID], id)
	return nil
}

// Recent returns a slice of len `quantity` (or fewer) nodes of the given type.
// These nodes are the most recent (by creation time) nodes of that type known
// to the store.
func (m *MemoryStore) Recent(nodeType fields.NodeType, quantity int) ([]forest.Node, error) {
	// highly inefficient implementation, but it should work for now
	candidates := make([]forest.Node, 0, quantity)
	for _, node := range m.Items {
		switch n := node.(type) {
		case *forest.Identity:
			if nodeType == fields.NodeTypeIdentity {
				candidates = append(candidates, n)
				sort.SliceStable(candidates, func(i, j int) bool {
					return candidates[i].(*forest.Identity).Created > candidates[j].(*forest.Identity).Created
				})
			}
		case *forest.Community:
			if nodeType == fields.NodeTypeCommunity {
				candidates = append(candidates, n)
				sort.SliceStable(candidates, func(i, j int) bool {
					return candidates[i].(*forest.Community).Created > candidates[j].(*forest.Community).Created
				})
			}
		case *forest.Reply:
			if nodeType == fields.NodeTypeReply {
				candidates = append(candidates, n)
				sort.SliceStable(candidates, func(i, j int) bool {
					return candidates[i].(*forest.Reply).Created > candidates[j].(*forest.Reply).Created
				})
			}
		}
	}
	if len(candidates) > quantity {
		candidates = candidates[:quantity]
	}
	return candidates, nil
}

R store_test.go => store/store_test.go +12 -11
@@ 1,15 1,16 @@
package forest_test
package store_test

import (
	"testing"

	forest "git.sr.ht/~whereswaldon/forest-go"
	"git.sr.ht/~whereswaldon/forest-go/fields"
	"git.sr.ht/~whereswaldon/forest-go/store"
	"git.sr.ht/~whereswaldon/forest-go/testutil"
)

func TestMemoryStore(t *testing.T) {
	s := forest.NewMemoryStore()
	s := store.NewMemoryStore()
	testStandardStoreInterface(t, s, "MemoryStore")
}



@@ 154,9 155,9 @@ func containsID(ids []*fields.QualifiedHash, id *fields.QualifiedHash) bool {
}

func TestCacheStore(t *testing.T) {
	s1 := forest.NewMemoryStore()
	s2 := forest.NewMemoryStore()
	c, err := forest.NewCacheStore(s1, s2)
	s1 := store.NewMemoryStore()
	s2 := store.NewMemoryStore()
	c, err := store.NewCacheStore(s1, s2)
	if err != nil {
		t.Errorf("Unexpected error constructing CacheStore: %v", err)
	}


@@ 164,7 165,7 @@ func TestCacheStore(t *testing.T) {
}

func TestCacheStoreDownPropagation(t *testing.T) {
	s1 := forest.NewMemoryStore()
	s1 := store.NewMemoryStore()
	id, _, com, rep := testutil.MakeReplyOrSkip(t)
	nodes := []forest.Node{id, com, rep}
	subrange := nodes[:len(nodes)-1]


@@ 173,8 174,8 @@ func TestCacheStoreDownPropagation(t *testing.T) {
			t.Skipf("Failed adding %v to %v", node, s1)
		}
	}
	s2 := forest.NewMemoryStore()
	if _, err := forest.NewCacheStore(s1, s2); err != nil {
	s2 := store.NewMemoryStore()
	if _, err := store.NewCacheStore(s1, s2); err != nil {
		t.Errorf("Unexpected error when constructing CacheStore: %v", err)
	}



@@ 190,7 191,7 @@ func TestCacheStoreDownPropagation(t *testing.T) {
}

func TestCacheStoreUpPropagation(t *testing.T) {
	base := forest.NewMemoryStore()
	base := store.NewMemoryStore()
	id, _, com, rep := testutil.MakeReplyOrSkip(t)
	nodes := []forest.Node{id, com, rep}
	subrange := nodes[:len(nodes)-1]


@@ 199,8 200,8 @@ func TestCacheStoreUpPropagation(t *testing.T) {
			t.Skipf("Failed adding %v to %v", node, base)
		}
	}
	cache := forest.NewMemoryStore()
	combined, err := forest.NewCacheStore(cache, base)
	cache := store.NewMemoryStore()
	combined, err := store.NewCacheStore(cache, base)
	if err != nil {
		t.Errorf("Unexpected error when constructing CacheStore: %v", err)
	}