~whereswaldon/forest-go

feb09985c341dfed82f9f7fabe237076b8a3265f — Chris Waldon 1 year, 11 months ago 57881ba + 661dbce
Merge branch 'master' into grove-add
A .github/workflows/issue-replication.yml => .github/workflows/issue-replication.yml +31 -0
@@ 0,0 1,31 @@
name: Issue Autoresponse

on:
  issues:
    types: [opened]

jobs:
  auto-response:
    runs-on: ubuntu-latest

    steps:
    - uses: derekprior/add-autoresponse@master
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        respondableId: ${{ github.event.issue.node_id }}
        response: "Hello! Thank you for your interest in Arbor!\nWe've chosen to mirror this repo to GitHub so that it's easier to find, but our issue tracking is done using [sourcehut](https://sourcehut.org).\nWe've automatically created a ticket in our sourcehut issue tracker with the contents of your issue. We'll follow up with you there! You can find your ticket [here!](https://todo.sr.ht/~whereswaldon/arbor-dev)\nThanks!"
        author: ${{ github.event.issue.user.login }}

  mirror:
    runs-on: ubuntu-latest

    steps:
    - uses: athorp96/sourcehut_issue_mirror@master
      with:
        title: ${{ github.event.issue.title }}
        body: ${{ github.event.issue.body }}
        submitter: ${{ github.event.issue.user.login }}
        tracker-owner: "~whereswaldon"
        tracker-name: "arbor-dev"
        oauth-token: ${{ secrets.SRHT_OAUTH_TOKEN }}

M fields/qualifieds.go => fields/qualifieds.go +13 -7
@@ 15,8 15,8 @@ const minSizeofQualified = sizeofDescriptor

// concrete qualified data types
type QualifiedHash struct {
	Descriptor HashDescriptor      `arbor:"order=0,recurse=serialize"`
	Blob                           `arbor:"order=1"`
	Descriptor HashDescriptor `arbor:"order=0,recurse=serialize"`
	Blob       `arbor:"order=1"`
}

const minSizeofQualifiedHash = sizeofHashDescriptor


@@ 87,6 87,12 @@ func (q *QualifiedHash) MarshalString() (string, error) {
	return string(s), e
}

// String returns the output of MarshalString, but does not return an error.
func (q *QualifiedHash) String() string {
	s, _ := q.MarshalString()
	return s
}

func (q *QualifiedHash) Validate() error {
	if err := q.Descriptor.Validate(); err != nil {
		return err


@@ 98,8 104,8 @@ func (q *QualifiedHash) Validate() error {
}

type QualifiedContent struct {
	Descriptor ContentDescriptor   `arbor:"order=0,recurse=serialize"`
	Blob                           `arbor:"order=1"`
	Descriptor ContentDescriptor `arbor:"order=0,recurse=serialize"`
	Blob       `arbor:"order=1"`
}

const minSizeofQualifiedContent = sizeofContentDescriptor


@@ 160,8 166,8 @@ func (q *QualifiedContent) Validate() error {
}

type QualifiedKey struct {
	Descriptor KeyDescriptor       `arbor:"order=0,recurse=serialize"`
	Blob                           `arbor:"order=1"`
	Descriptor KeyDescriptor `arbor:"order=0,recurse=serialize"`
	Blob       `arbor:"order=1"`
}

const minSizeofQualifiedKey = sizeofKeyDescriptor


@@ 216,7 222,7 @@ func (q *QualifiedKey) AsEntity() (*openpgp.Entity, error) {

type QualifiedSignature struct {
	Descriptor SignatureDescriptor `arbor:"order=0,recurse=serialize"`
	Blob                           `arbor:"order=1"`
	Blob       `arbor:"order=1"`
}

const minSizeofQualifiedSignature = sizeofSignatureDescriptor

M grove/grove.go => grove/grove.go +81 -6
@@ 91,11 91,11 @@ func NewWithFS(fs FS) (*Grove, error) {
// Get searches the grove for a node with the given id. It returns the node if it was
// found, a boolean indicating whether it was found, and an error (if there was a
// problem searching for the node).
func (g *Grove) Get(nodeID *fields.QualifiedHash) (forest.Node, bool, error) {
	filename, err := nodeID.MarshalString()
	if err != nil {
		return nil, false, fmt.Errorf("failed determining file name for node: %w", err)
	}
// The returned `present` will never be true unless the returned `node` holds an
// actual node struct. If the file holding a node exists on disk but was unable
// to be opened, read, or parsed, `present` will still be false.
func (g *Grove) Get(nodeID *fields.QualifiedHash) (node forest.Node, present bool, err error) {
	filename := nodeID.String()
	file, err := g.Open(filename)
	// if the file doesn't exist, just return false with no error
	if errors.Is(err, os.ErrNotExist) {


@@ 109,7 109,7 @@ func (g *Grove) Get(nodeID *fields.QualifiedHash) (forest.Node, bool, error) {
	if err != nil {
		return nil, false, fmt.Errorf("failed reading bytes from \"%s\": %w", filename, err)
	}
	node, err := forest.UnmarshalBinaryNode(b)
	node, err = forest.UnmarshalBinaryNode(b)
	if err != nil {
		return nil, false, fmt.Errorf("failed unmarshalling node from \"%s\": %w", filename, err)
	}


@@ 140,3 140,78 @@ func (g *Grove) Add(node forest.Node) error {
	}
	return nil
}

// GetIdentity returns an Identity node with the given ID (if it is present
// in the grove). This operation may be faster than using Get, as the grove
// may be able to do less search work when it knows the type of node you're
// looking for in advance.
//
// BUG(whereswaldon): The current implementation may return nodes of the
// wrong NodeType if they match the provided ID
func (g *Grove) GetIdentity(id *fields.QualifiedHash) (forest.Node, bool, error) {
	// this naiive implementation is not efficient, but works as a short-term
	// thing.
	//
	// TODO: change the on-disk representation so that operations like this can
	// be fast (store different node types in different directories, etc...)
	return g.Get(id)
}

// GetCommunity returns an Community node with the given ID (if it is present
// in the grove). This operation may be faster than using Get, as the grove
// may be able to do less search work when it knows the type of node you're
// looking for in advance.
//
// BUG(whereswaldon): The current implementation may return nodes of the
// wrong NodeType if they match the provided ID
func (g *Grove) GetCommunity(id *fields.QualifiedHash) (forest.Node, bool, error) {
	// this naiive implementation is not efficient, but works as a short-term
	// thing.
	//
	// TODO: change the on-disk representation so that operations like this can
	// be fast (store different node types in different directories, etc...)
	return g.Get(id)
}

// GetConversation returns an Conversation node with the given ID (if it is present
// in the grove). This operation may be faster than using Get, as the grove
// may be able to do less search work when it knows the type of node you're
// looking for and its parent node in advance.
//
// BUG(whereswaldon): The current implementation may return nodes of the
// wrong NodeType if they match the provided ID
func (g *Grove) GetConversation(communityID, conversationID *fields.QualifiedHash) (forest.Node, bool, error) {
	// this naiive implementation is not efficient, but works as a short-term
	// thing.
	//
	// TODO: change the on-disk representation so that operations like this can
	// be fast (store different node types in different directories, etc...)
	return g.Get(conversationID)
}

// GetReply returns an Reply node with the given ID (if it is present
// in the grove). This operation may be faster than using Get, as the grove
// may be able to do less search work when it knows the type of node you're
// looking for and its parent community and conversation node in advance.
//
// BUG(whereswaldon): The current implementation may return nodes of the
// wrong NodeType if they match the provided ID
func (g *Grove) GetReply(communityID, conversationID, replyID *fields.QualifiedHash) (forest.Node, bool, error) {
	// this naiive implementation is not efficient, but works as a short-term
	// thing.
	//
	// TODO: change the on-disk representation so that operations like this can
	// be fast (store different node types in different directories, etc...)
	return g.Get(replyID)
}

// CopyInto copies all nodes from the store into the provided store.
//
// BUG(whereswaldon): this method is not yet implemented. It requires
// more extensive file manipulation than other Grove methods (listing
// directory contents) and has therefore been deprioritized in favor
// of the functionality that can be implemented simply. However, it is
// implementable, and should be done as soon as is feasible.
func (g *Grove) CopyInto(other forest.Store) error {
	return fmt.Errorf("method CopyInto() is not currently implemented on Grove")
}

M grove/grove_test.go => grove/grove_test.go +76 -6
@@ 222,12 222,7 @@ func (tnb *testNodeBuilder) newReplyFile(content string) (*forest.Reply, *fakeFi
	if err != nil {
		tnb.T.Errorf("Failed marshalling test reply node: %v", err)
	}
	id := reply.ID()
	nodeID, err := id.MarshalString()
	if err != nil {
		tnb.T.Errorf("Failed to marshal node id: %v", err)
	}
	return reply, newFakeFile(nodeID, b)
	return reply, newFakeFile(reply.ID().String(), b)
}

func TestCreateEmptyGrove(t *testing.T) {


@@ 279,10 274,83 @@ func TestGroveGet(t *testing.T) {
	}
}

func TestGroveGetErrorReadingFile(t *testing.T) {
	fs := newFakeFS()
	fakeNodeBuilder := NewNodeBuilder(t)
	reply, replyFile := fakeNodeBuilder.newReplyFile("test content")
	errReplyFile := NewErrFile(replyFile)
	g, err := grove.NewWithFS(fs)
	if err != nil {
		t.Errorf("Failed constructing grove: %v", err)
	}

	// add node to fs, now should be discoverable
	fs.files[errReplyFile.Name()] = errReplyFile
	errReplyFile.error = os.ErrClosed

	// no nodes in fs, make sure we get nothing
	if node, present, err := g.Get(reply.ID()); err == nil {
		t.Errorf("Expected error reading file to be propagated upward, got nil")
	} else if present {
		t.Errorf("Grove indicated that a node was present when it could not be read")
	} else if node != nil {
		t.Errorf("Grove returned a node when the requested node was unreadable")
	}
}

func TestGroveGetErrorUnmarshallingFile(t *testing.T) {
	fs := newFakeFS()
	fakeNodeBuilder := NewNodeBuilder(t)
	reply, replyFile := fakeNodeBuilder.newReplyFile("test content")
	replyFile.Reset()
	_, err := replyFile.Write([]byte("this is not an arbor node"))
	if err != nil {
		t.Skipf("Unable to write test data into node file: %v", err)
	}
	g, err := grove.NewWithFS(fs)
	if err != nil {
		t.Errorf("Failed constructing grove: %v", err)
	}

	// add node to fs, now should be discoverable
	fs.files[replyFile.Name()] = replyFile

	// no nodes in fs, make sure we get nothing
	if node, present, err := g.Get(reply.ID()); err == nil {
		t.Errorf("Expected error unmarshalling file to be propagated upward, got nil")
	} else if present {
		t.Errorf("Grove indicated that a node was present when it could not be unmarshalled")
	} else if node != nil {
		t.Errorf("Grove returned a node when the requested node was unparsable")
	}
}

func TestGroveGetErrorOpeningFile(t *testing.T) {
	fs := newFakeFS()
	eFS := newErrFS(fs)
	fakeNodeBuilder := NewNodeBuilder(t)
	reply, _ := fakeNodeBuilder.newReplyFile("test content")
	g, err := grove.NewWithFS(eFS)
	if err != nil {
		t.Errorf("Failed constructing grove: %v", err)
	}
	eFS.error = os.ErrPermission

	// no nodes in fs, make sure we get nothing
	if node, present, err := g.Get(reply.ID()); err == nil {
		t.Errorf("Expected error accessing file to be propagated upward, got nil")
	} else if present {
		t.Errorf("Grove indicated that a node was present when it could not be opened")
	} else if node != nil {
		t.Errorf("Grove returned a node when the requested node was inaccessible")
	}
}

func TestGroveAdd(t *testing.T) {
	fs := newFakeFS()
	fakeNodeBuilder := NewNodeBuilder(t)
	reply, _ := fakeNodeBuilder.newReplyFile("test content")

	g, err := grove.NewWithFS(fs)
	if err != nil {
		t.Errorf("Failed constructing grove: %v", err)


@@ 298,10 366,12 @@ func TestGroveAddFailToWrite(t *testing.T) {
	fakeNodeBuilder := NewNodeBuilder(t)
	reply, replyFile := fakeNodeBuilder.newReplyFile("test content")
	eFile := NewErrFile(replyFile)

	g, err := grove.NewWithFS(fs)
	if err != nil {
		t.Errorf("Failed constructing grove: %v", err)
	}

	fs.files[eFile.Name()] = eFile
	eFile.error = os.ErrClosed


M store.go => store.go +0 -11
@@ 8,7 8,6 @@ import (
)

type Store interface {
	Size() (int, error)
	CopyInto(Store) error
	Get(*fields.QualifiedHash) (Node, bool, error)
	GetIdentity(*fields.QualifiedHash) (Node, bool, error)


@@ 34,10 33,6 @@ func NewMemoryStore() *MemoryStore {
	}
}

func (m *MemoryStore) Size() (int, error) {
	return len(m.Items), nil
}

func (m *MemoryStore) CopyInto(other Store) error {
	for _, node := range m.Items {
		if err := other.Add(node); err != nil {


@@ 179,12 174,6 @@ func NewCacheStore(cache, back Store) (*CacheStore, error) {
	return &CacheStore{cache, back}, nil
}

// Size returns the effective size of this CacheStore, which is the size of the
// Back Store.
func (m *CacheStore) Size() (int, error) {
	return m.Back.Size()
}

// Get returns the requested node if it is present in either the Cache or the Back Store.
// If the cache is missed by the backing store is hit, the node will automatically be
// added to the cache.

M store_test.go => store_test.go +1 -11
@@ 13,11 13,6 @@ func TestMemoryStore(t *testing.T) {
}

func testStandardStoreInterface(t *testing.T, s forest.Store, storeImplName string) {
	if size, err := s.Size(); size != 0 {
		t.Errorf("Expected new %s to have size 0, had %d", storeImplName, size)
	} else if err != nil {
		t.Errorf("Expected new %s Size() to succeed, failed with %s", storeImplName, err)
	}
	// create three test nodes, one of each type
	identity, _, community, reply := MakeReplyOrSkip(t)
	nodes := []forest.Node{identity, community, reply}


@@ 49,15 44,10 @@ func testStandardStoreInterface(t *testing.T, s forest.Store, storeImplName stri
	}

	// add each node
	for count, i := range nodes {
	for _, i := range nodes {
		if err := s.Add(i); err != nil {
			t.Errorf("%s Add() should not err on Add(): %s", storeImplName, err)
		}
		if size, err := s.Size(); err != nil {
			t.Errorf("%s Size() should never error, got %s", storeImplName, err)
		} else if size != count+1 {
			t.Errorf("%s Size() should be %d after %d Add()s, got %d", storeImplName, count+1, count+1, size)
		}
	}

	// map each node to the getters that should be successful in fetching it