~badt/glavar

ref: 5fdf23346726bef4f92960b2946e7f34e21b4410 glavar/fthinoporo/index.go -rw-r--r-- 1.2 KiB
5fdf2334Ian P Badtrousers fthinoporo: more insight 4 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
package main

import (
	"database/sql"
	"fmt"
	"math/rand"
	"runtime"
	"strconv"
	"time"

	"github.com/blevesearch/bleve"
)

func index() {
	rand.Seed(time.Now().UnixNano())

	mapping := bleve.NewIndexMapping()
	idx, err := bleve.New("index", mapping)
	if err != nil {
		panic(err)
	}
	defer idx.Close()

	db, err := sql.Open("sqlite3", "logec.sqlite")
	if err != nil {
		panic(err)
	}
	defer db.Close()

	data, err := db.Query("SELECT * FROM logec ORDER BY time ASC")
	if err != nil {
		panic(err)
	}

	N, pipe := 0, make(chan M, 1000)

	cpu := runtime.NumCPU()
	fmt.Println("cpu count:", cpu)

	var start time.Time
	for i := 0; i < cpu; i++ {
		go func(i int) {
			n := 0
			for m := range pipe {
				idx.Index(strconv.FormatInt(m.T, 10), m)
				n++
				N++

				if rand.Intn(1000) == 7 {
					fmt.Printf("[%v] indexed:%d, total:%d\n",
						time.Now().Sub(start), n, N)
				}
			}
		}(i)
	}

	fmt.Println("loading data")
	start = time.Now()
	for data.Next() {
		var m M
		err = data.Scan(&m.T, &m.Login, &m.Text)
		if err != nil {
			fmt.Println("!!", err)
			return
		}
		pipe <- m
		if rand.Intn(1000) == 7 {
			t := time.Unix(0, m.T).Format(time.RFC822)
			fmt.Println(t)
		}
	}

	close(pipe)
}