~mendelmaleh/pfin

ref: 95ffe938fcda9ea79264ad9b3fbfd09059af3b17 pfin/util/dir.go -rw-r--r-- 1.5 KiB
95ffe938Mendel E Parallelize directory parsing 3 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
package util

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"sync"

	"git.sr.ht/~mendelmaleh/pfin"
)

type result struct {
	s  string
	tx []pfin.Transaction
}

type ErrNoMatches struct {
	path string
}

func (e ErrNoMatches) Error() string {
	return fmt.Sprintf("pfin/parser/util: no matches for path %q", e.path)
}

func ParseDir(acc pfin.Account, root string) ([]pfin.Transaction, error) {
	var txns []pfin.Transaction

	filetype, err := pfin.Filetype(acc.Type)
	if err != nil {
		return txns, err
	}

	path := filepath.Join(root, acc.Name)

	if path[len(path)-1] != filepath.Separator {
		path += string(filepath.Separator)
	}

	matches, err := filepath.Glob(path + "*." + filetype)
	if err != nil {
		return txns, err
	}

	if len(matches) == 0 {
		return txns, ErrNoMatches{path}
	}

	ch := make(chan result, len(matches))
	cherr := make(chan error, 1)

	var wg sync.WaitGroup

	for _, f := range matches {
		wg.Add(1)

		go func(f string) {
			defer wg.Done()

			file, err := os.ReadFile(f)
			if err != nil {
				cherr <- err
				return
			}

			tx, err := pfin.Parse(acc, filepath.Base(f), file)
			if err != nil {
				cherr <- err
			}

			ch <- result{f, tx}
		}(f)
	}

	wg.Wait()
	close(ch)

	files := make(map[string][]pfin.Transaction)

	for len(files) < len(matches) {
		select {
		case err := <-cherr:
			return txns, err
		case res := <-ch:
			files[res.s] = res.tx
			// txns = append(txns, tx...)
		}
	}

	// sort oldest first
	sort.Strings(matches)
	for _, f := range matches {
		txns = append(txns, files[f]...)
	}

	return txns, nil
}