~badt/glavar

7f094d75b4139928e94262b0e81681664867c4c8 — Ian P Badtrousers 3 months ago 7d25ca0 master
glavar: authumnuscha got fucked
5 files changed, 178 insertions(+), 2 deletions(-)

A fthinoporo/.gitignore
A fthinoporo/main.go
M go.mod
M go.sum
M user.go
A fthinoporo/.gitignore => fthinoporo/.gitignore +2 -0
@@ 0,0 1,2 @@
logec.sqlite
fthinoporo

A fthinoporo/main.go => fthinoporo/main.go +153 -0
@@ 0,0 1,153 @@
package main

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"jaytaylor.com/html2text"
)

const (
	sql_CREATE_TABLE_logec = `
CREATE TABLE "logec" (
	"time"	INTEGER NOT NULL,
	"login"	TEXT,
	"text"	TEXT,
	PRIMARY KEY("time")
);`

	sql_INSERT_MESSAGE = `
INSERT INTO "logec" ("time", "login", "text") VALUES (?, ?, ?);`
)

type M struct {
	T     int64  `sql:"time"`
	Login string `sql:"login"`
	Text  string `sql:"text"`
}

func feed(bus chan M) {
	path := os.Getenv("LOGEC")
	for i := 1; ; i++ {
		b, err := ioutil.ReadFile(fmt.Sprintf("%s/%d.html", path, i))
		if err != nil {
			break
		}
		tr := strings.Split(string(b), "</tr>")
		for _, cellb := range tr {
			if strings.Contains(cellb, "data-total-pages") {
				continue
			}
			td := strings.Split(cellb, "</td>")
			if len(td) == 0 || td[0] == "" {
				continue
			}

			t, login, opus := gett(td[0]), get(td[1]), get(td[2])
			opus, _ = html2text.FromString(opus, html2text.Options{})
			bus <- M{t.UnixNano(), login, opus}
		}
	}
	close(bus)
}

func main() {
	os.Remove("logec.sqlite")
	db, err := sql.Open("sqlite3", "logec.sqlite")
	if err != nil {
		panic(err)
	}
	defer db.Close()
	if _, err := db.Exec(sql_CREATE_TABLE_logec); err != nil {
		panic(err)
	}

	bus, auto := make(chan M), make(chan M)
	go feed(bus)
	go func() {
		dedup := make([]M, 1, 200)
		for m := range bus {
			tail := dedup[len(dedup)-1]
			// duplicate
			if tail.Text == m.Text {
				continue
			}
			// to be ordered
			if tail.T == m.T {
				dedup = append(dedup, m)
				continue
			}

			if len(dedup) > 1 {
				qsize := time.Second / time.Duration(len(dedup))
				for i, mm := range dedup {
					if mm.T == 0 {
						break
					}
					mm.T -= int64(time.Duration(i) * qsize)
					auto <- mm
				}
				dedup = dedup[:1]
				dedup[0] = m
				continue
			}

			if dedup[0].T != 0 {
				auto <- dedup[0]
			}
			dedup[0] = m
		}
		if len(dedup) > 1 {
			qsize := time.Second / time.Duration(len(dedup))
			for i, mm := range dedup {
				if mm.T == 0 {
					break
				}
				mm.T -= int64(time.Duration(i) * qsize)
				auto <- mm
			}
		}
		close(auto)
	}()

	tx, _ := db.Begin()
	stmt, err := tx.Prepare(sql_INSERT_MESSAGE)
	if err != nil {
		panic(err)
	}
	defer stmt.Close()

	pushed := map[int64]bool{}
	for m := range auto {
		if _, ok := pushed[m.T]; ok {
			continue
		}
		_, err = stmt.Exec(m.T, m.Login, m.Text)
		if err != nil {
			fmt.Println("!", m, err)
			continue
		}
		pushed[m.T] = true
	}
	tx.Commit()
}

var timeRx = regexp.MustCompile(`data-param='([^']+)'`)

func gett(s string) time.Time {
	m := timeRx.FindAllStringSubmatch(s, -1)
	tstr := m[0][1]
	t, _ := time.Parse("2006-01-02 15:04", tstr)
	return t
}

func get(s string) string {
	y := strings.Split(s, "<td class='  '>")
	return strings.TrimSpace(strings.Join(y[1:], ""))
}

M go.mod => go.mod +5 -0
@@ 4,7 4,12 @@ go 1.17

require (
	github.com/bluele/gcache v0.0.2
	github.com/mattn/go-sqlite3 v1.14.8 // indirect
	github.com/olekukonko/tablewriter v0.0.5 // indirect
	github.com/pkg/errors v0.9.1
	github.com/sethvargo/go-retry v0.1.0 // indirect
	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
	golang.org/x/net v0.0.0-20210716203947-853a461950ff // indirect
	gopkg.in/tucnak/telebot.v3 v3.0.0-20210705112529-7e3a68249a0b
	jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7
)

M go.sum => go.sum +16 -0
@@ 10,6 10,12 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=


@@ 19,15 25,23 @@ github.com/sethvargo/go-retry v0.1.0 h1:8sPqlWannzcReEcYjHSNw9becsiYudcwTD7CasGj
github.com/sethvargo/go-retry v0.1.0/go.mod h1:JzIOdZqQDNpPkQDmcqgtteAcxFLtYpNF/zJCM1ysDg8=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
golang.org/x/net v0.0.0-20210716203947-853a461950ff h1:j2EK/QoxYNBsXI4R7fQkkRUk8y6wnOBI+6hgPdP/6Ds=
golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=


@@ 40,3 54,5 @@ gopkg.in/tucnak/telebot.v3 v3.0.0-20210703231635-83bd8dd89d7f/go.mod h1:rQkLEeZ6
gopkg.in/tucnak/telebot.v3 v3.0.0-20210705112529-7e3a68249a0b h1:zIsZMLZJoGp09RXcAKnlrx5WeMw+QyNEwmjMrkRrcPI=
gopkg.in/tucnak/telebot.v3 v3.0.0-20210705112529-7e3a68249a0b/go.mod h1:rQkLEeZ6z+UenbjlO1GjW24ZIZ6yJWqv92jcXpGpiR8=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:mub0MmFLOn8XLikZOAhgLD1kXJq8jgftSrrv7m00xFo=
jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:OxvTsCwKosqQ1q7B+8FwXqg4rKZ/UG9dUW+g/VL2xH4=

M user.go => user.go +2 -2
@@ 253,10 253,10 @@ func (u *User) poll(subsite string) error {
			}
			rate := value.(int) + 1
			rates.SetWithExpire(url, rate, rateWindow)
			if rate > 1 {
			if rate > 3 {
				nopreview = true
			}
			if rate > 3 {
			if rate > 5 {
				noshow = true
				break
			}