~mariusor/fedbox-spammy

fedbox-spammy/loader.go -rw-r--r-- 2.3 KiB
d9150b40 — Marius Orcsik Made the timeouts for requests uniform at 5 seconds 4 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
package spammy

import (
	"context"
	ap "github.com/go-ap/activitypub"
	"github.com/go-ap/client"
	"golang.org/x/sync/errgroup"
	"sync"
	"time"
)

type loader struct {
	f     *client.C
	queue chan ap.IRI
	res   chan ap.Item
	done  chan bool
}

func (l loader) loadFn(ctx context.Context, i ap.IRI) func() error {
	return func() error {
		dtx, _ := context.WithTimeout(ctx, 5*time.Second)
		//defer cancelFn()

		ob, err := l.f.CtxLoadIRI(dtx, i)
		if err != nil {
			return err
		}
		var (
			next  ap.Item
			items ap.ItemCollection
		)
		if ob.GetType() == ap.OrderedCollectionType {
			ap.OnOrderedCollection(ob, func(col *ap.OrderedCollection) error {
				// NOTE(marius): this wastes a request
				next = col.First
				return nil
			})
		}
		if ob.GetType() == ap.OrderedCollectionPageType {
			ap.OnOrderedCollectionPage(ob, func(col *ap.OrderedCollectionPage) error {
				next = col.Next
				items = col.OrderedItems
				return nil
			})
		}
		if items.Count() > 0 {
			for _, it := range items.Collection() {
				l.res <- it
			}
		}
		if next != nil && !next.GetLink().Equals(i, false) {
			l.queue <- next.GetLink()
		} else {
			l.done <- true
		}
		return nil
	}
}

func (l loader) wait(ctx context.Context, cancelFn func()) (map[ap.IRI]ap.Item, []error) {
	result := make(map[ap.IRI]ap.Item)
	errors := make([]error, 0)

	g, gtx := errgroup.WithContext(ctx)
	m := sync.Mutex{}
	defer cancelFn()

exit:
	for {
		select {
		case done := <-l.done:
			if done {
				cancelFn()
				break exit
			}
		case <-ctx.Done():
			cancelFn()
			break exit
		case <-gtx.Done():
			cancelFn()
			break exit
		case i := <-l.queue:
			g.Go(l.loadFn(gtx, i))
		case it := <-l.res:
			m.Lock()
			if it.GetType() != ap.TombstoneType {
				if _, ok := result[it.GetLink()]; !ok {
					result[it.GetLink()] = it
				}
			}
			m.Unlock()
		}
	}
	if err := g.Wait(); err != nil {
		errors = append(errors, err)
	}
	return result, errors
}

func load(iri ap.IRI, concurrent int) (map[ap.IRI]ap.Item, []error) {
	l := loader{
		f:     client.New(client.SkipTLSValidation(true), client.SetErrorLogger(ErrFn)),
		queue: make(chan ap.IRI, concurrent),
		res:   make(chan ap.Item),
		done:  make(chan bool, 1),
	}

	ctx, cancelFn := context.WithCancel(context.TODO())
	//defer cancelFn()

	l.queue <- iri

	return l.wait(ctx, cancelFn)
}