~hokiegeek/seculardb

a8de10baa299b694c22209d9df1ff84ae156e5ef — HokieGeek 6 months ago 35780c9
Updated to new version of htmlscrape
3 files changed, 5 insertions(+), 5 deletions(-)

M go.mod
M go.sum
M scraper.go
M go.mod => go.mod +1 -1
@@ 3,7 3,7 @@ module git.sr.ht/~hokiegeek/seculardb
go 1.12

require (
	git.sr.ht/~hokiegeek/htmlscrape v0.3.3
	git.sr.ht/~hokiegeek/htmlscrape v0.4.0
	github.com/dustin/go-humanize v1.0.0
	github.com/spf13/cobra v0.0.5
	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80

M go.sum => go.sum +2 -2
@@ 1,5 1,5 @@
git.sr.ht/~hokiegeek/htmlscrape v0.3.3 h1:MhENqe7JW9nAryEnx0tAKF14V+G82MKf/JnjJR9iNng=
git.sr.ht/~hokiegeek/htmlscrape v0.3.3/go.mod h1:64bIE/nqM1IlgD19Lxtzbr9oIhvX+45qTbGceA6cLcc=
git.sr.ht/~hokiegeek/htmlscrape v0.4.0 h1:TY5GipSScKDM8kwEF23Kmj6aTUIn8+vOV3pmKH/g7X4=
git.sr.ht/~hokiegeek/htmlscrape v0.4.0/go.mod h1:64bIE/nqM1IlgD19Lxtzbr9oIhvX+45qTbGceA6cLcc=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=

M scraper.go => scraper.go +2 -2
@@ 28,7 28,7 @@ func Build() (db DB, err error) {
			defer wg.Done()

			colMatcher := func(n *html.Node, name string) *html.Node {
				return htmlscrape.FindNode(n, htmlscrape.NewNodeMatcher().Type(html.ElementNode).Atom(atom.Td).Attr("class", name))
				return htmlscrape.FindNode(n, htmlscrape.NewNodeMatcher().Elem().Atom(atom.Td).Attr("class", name))
			}

			col := func(n *html.Node, name string) string {


@@ 40,7 40,7 @@ func Build() (db DB, err error) {
					case c.Type == html.TextNode && strings.Contains(c.Data, "wp-content"):
						fallthrough
					case c.Type == html.ElementNode && c.DataAtom == atom.Center:
						b := htmlscrape.FindNode(td, htmlscrape.NewNodeMatcher().Type(html.ElementNode).Atom(atom.B))
						b := htmlscrape.FindNode(td, htmlscrape.NewNodeMatcher().Elem().Atom(atom.B))
						switch {
						case b == nil:
							buf.WriteString(c.FirstChild.Data)