~badt/glavar

ref: 6a19d38e5b86c50505c328d0b100e77fd5f2db96 glavar/fthinoporo/index.go -rw-r--r-- 1.2 KiB
6a19d38eIan P Badtrousers fthinoporo: multicore batching 4 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
package main

import (
	"database/sql"
	"fmt"
	"runtime"
	"strconv"
	"sync"

	"github.com/blevesearch/bleve"
)

func index() {
	mapping := bleve.NewIndexMapping()
	idx, err := bleve.New("index", mapping)
	if err != nil {
		panic(err)
	}
	defer idx.Close()

	db, err := sql.Open("sqlite3", "logec.sqlite")
	if err != nil {
		panic(err)
	}
	defer db.Close()

	data, err := db.Query("SELECT * FROM logec ORDER BY time DESC")
	if err != nil {
		panic(err)
	}

	const batchSize = 2500

	cpu := runtime.NumCPU()
	batches := make(chan []M, cpu)
	wg := &sync.WaitGroup{}
	for i := 0; i < cpu; i++ {
		go func(i int) {
			wg.Add(1)
			defer wg.Done()
			for messages := range batches {
				b := idx.NewBatch()
				for _, m := range messages {
					b.Index(strconv.FormatInt(m.T, 10), m)
				}
				idx.Batch(b)
				fmt.Printf("[%d] batch %d\n", i, len(messages))
			}
		}(i)
	}

	A := make([]M, 0, batchSize)
	for data.Next() {
		var m M
		err = data.Scan(&m.T, &m.Login, &m.Text)
		if err != nil {
			fmt.Println("!!", err)
			return
		}

		A = append(A, m)

		if len(A) == cap(A) {
			batches <- A
			A = make([]M, 0, batchSize)
		}
	}

	if len(A) != 0 {
		batches <- A
	}

	close(batches)
	wg.Wait()
}