~hokiegeek/seculardb

35780c9c8c7feb23a4cfcdd65ad3e93617fe4546 — HokieGeek 2 years ago 89be52d v1.5.0
Added link, made the HTML page look nicer
5 files changed, 116 insertions(+), 58 deletions(-)

M db.go
M deploy-function.sh
M scraper.go
R function.go => serve.go
R function_test.go => serve_test.go
M db.go => db.go +49 -37
@@ 7,8 7,10 @@ import (
	"github.com/dustin/go-humanize"
)

// Rating enumerates the nominal rating values
type Rating int

// These are the basic ratings
const (
	RatingUnconfirmed Rating = iota
	RatingNotSecular


@@ 37,18 39,19 @@ func (r Rating) String() string {
	}
}

// Entry encapsulates a single row in the Guide
type Entry struct {
	Name        string
	Rating      Rating
	GradeLevels []string
	Subjects    []string
	Description string
	Name, Description, URL string
	GradeLevels, Subjects  []string
	Rating                 Rating
}

// DB encapsulates the Guide rows
type DB struct {
	Entries []Entry
}

// EntryMatcher provides a builder to specify criteria for filtering Entries
type EntryMatcher struct {
	// keywords        []string
	rating          int


@@ 57,55 60,62 @@ type EntryMatcher struct {
}

/*
func (b *EntryMatcher) Keyword(v string) *EntryMatcher {
	b.keywords = append(b.keywords, v)
	return b
func (m *EntryMatcher) Keyword(v string) *EntryMatcher {
	m.keywords = append(m.keywords, v)
	return m
}
*/

func (b *EntryMatcher) Rating(v int) *EntryMatcher {
	b.rating = v
	return b
// Rating is used to add a minimum entry rating to match on
func (m *EntryMatcher) Rating(v int) *EntryMatcher {
	m.rating = v
	return m
}

func (b *EntryMatcher) Grade(v int) *EntryMatcher {
	b.grades = append(b.grades, v)
	return b
// Grade is used to add a grade level to match on
func (m *EntryMatcher) Grade(v int) *EntryMatcher {
	m.grades = append(m.grades, v)
	return m
}

func (b *EntryMatcher) Grades(v []int) *EntryMatcher {
	b.grades = v
	return b
// Grades is used to add a slice of grade levels to match on
func (m *EntryMatcher) Grades(v []int) *EntryMatcher {
	m.grades = v
	return m
}

func (b *EntryMatcher) Name(v string) *EntryMatcher {
	b.names = append(b.names, v)
	return b
// Name is used to add a name to match on
func (m *EntryMatcher) Name(v string) *EntryMatcher {
	m.names = append(m.names, v)
	return m
}

func (b *EntryMatcher) Names(v []string) *EntryMatcher {
	b.names = v
	return b
// Names is used to add a slice of names to match on
func (m *EntryMatcher) Names(v []string) *EntryMatcher {
	m.names = v
	return m
}

func (b *EntryMatcher) Subject(v string) *EntryMatcher {
	b.subjects = append(b.subjects, v)
	return b
// Subject is used to add a subject to match on
func (m *EntryMatcher) Subject(v string) *EntryMatcher {
	m.subjects = append(m.subjects, v)
	return m
}

func (b *EntryMatcher) Subjects(v []string) *EntryMatcher {
	b.subjects = v
	return b
// Subjects is used to add a slice of subjects to match on
func (m *EntryMatcher) Subjects(v []string) *EntryMatcher {
	m.subjects = v
	return m
}

func (b *EntryMatcher) matches(e Entry) (_ bool) {
	if e.Rating < Rating(b.rating) {
func (m *EntryMatcher) matches(e Entry) (_ bool) {
	if e.Rating < Rating(m.rating) {
		return
	}

	if len(b.grades) > 0 {
	if len(m.grades) > 0 {
		var found bool
		for _, fgrade := range b.grades {
		for _, fgrade := range m.grades {
			for _, grade := range e.GradeLevels {
				switch g := strings.ToLower(grade); {
				case g == "pre-k" && fgrade == -1:


@@ 122,9 132,9 @@ func (b *EntryMatcher) matches(e Entry) (_ bool) {
		}
	}

	if len(b.names) > 0 {
	if len(m.names) > 0 {
		var found bool
		for _, fname := range b.names {
		for _, fname := range m.names {
			if strings.Contains(strings.ToLower(e.Name), strings.ToLower(fname)) {
				found = true
			}


@@ 134,9 144,9 @@ func (b *EntryMatcher) matches(e Entry) (_ bool) {
		}
	}

	if len(b.subjects) > 0 {
	if len(m.subjects) > 0 {
		var found bool
		for _, fsubject := range b.subjects {
		for _, fsubject := range m.subjects {
			fsubject = strings.ToLower(fsubject)
			for _, subject := range e.Subjects {
				if strings.Contains(strings.ToLower(subject), fsubject) {


@@ 152,10 162,12 @@ func (b *EntryMatcher) matches(e Entry) (_ bool) {
	return true
}

// NewMatcher creates a new EntryMatcher instance
func NewMatcher() *EntryMatcher {
	return new(EntryMatcher)
}

// Filter returns a copy of a DB object with only the Entries which matched against the EntryMatcher
func Filter(db DB, matcher *EntryMatcher) (filtered DB, err error) {
	filtered.Entries = make([]Entry, 0)


M deploy-function.sh => deploy-function.sh +1 -1
@@ 1,2 1,2 @@
#!/bin/sh
gcloud --project ${1} functions deploy seculardb --memory 128MB --runtime go111 --source . --entry-point Function --trigger-http
gcloud --project ${1} functions deploy seculardb --memory 128MB --runtime go111 --source . --entry-point Serve --trigger-http

M scraper.go => scraper.go +20 -1
@@ 12,8 12,10 @@ import (
	"git.sr.ht/~hokiegeek/htmlscrape"
)

// GuideURL is the URL for the WP page that has the Secular Homeschool Guide
const GuideURL = "https://www.secularhomeschooler.com/secular-homeschool-guide/"

// Build creates a DB object out of the page
func Build() (db DB, err error) {
	db.Entries = make([]Entry, 0)



@@ 25,8 27,12 @@ func Build() (db DB, err error) {
		go func() {
			defer wg.Done()

			colMatcher := func(n *html.Node, name string) *html.Node {
				return htmlscrape.FindNode(n, htmlscrape.NewNodeMatcher().Type(html.ElementNode).Atom(atom.Td).Attr("class", name))
			}

			col := func(n *html.Node, name string) string {
				td := htmlscrape.FindNode(n, htmlscrape.NewNodeMatcher().Type(html.ElementNode).Atom(atom.Td).Attr("class", name))
				td := colMatcher(n, name)

				var buf bytes.Buffer
				for c := td.FirstChild; c != nil; c = c.NextSibling {


@@ 54,6 60,18 @@ func Build() (db DB, err error) {
				gradeLevels := col(tr, "column-3") // []string?
				subjects := col(tr, "column-4")    // []string
				desc := col(tr, "column-5")
				linkNode := htmlscrape.FindNode(colMatcher(tr, "column-5"), htmlscrape.NewNodeMatcher().Atom(atom.A))
				var link string
				if linkNode != nil {
					for _, attr := range linkNode.Attr {
						if attr.Key == "href" {
							link = attr.Val
							if !strings.HasPrefix(link, "http") {
								link = "http://" + link
							}
						}
					}
				}

				ratingStr := strings.ToLower(col(tr, "column-2"))
				var rating int


@@ 89,6 107,7 @@ func Build() (db DB, err error) {
					GradeLevels: strings.Split(strings.Replace(gradeLevels, " ", "", -1), ","),
					Subjects:    strings.Split(strings.Replace(subjects, " ", "", -1), ","),
					Description: desc,
					URL:         link,
				}
				mu.Lock()
				db.Entries = append(db.Entries, entry)

R function.go => serve.go +44 -17
@@ 1,8 1,9 @@
// Package seculardb contains an HTTP Cloud Function.
package seculardb

import (
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"strconv"


@@ 10,8 11,19 @@ import (
)

var pageTmpl = `<html>
	<head>
		<title>Filterable Secular Homeschooler Guide</title>
		<style>
		th {
			border-bottom: 3px solid #777;
		}
		tr:nth-child(even) {
			background-color: #c0c0c0;
		}
		</style>
	</head>
	<body>
		<div><a href="` + GuideURL + `">` + GuideURL + `</a></div>
		<div><a href="` + GuideURL + `">` + GuideURL + `</a><span style="float: right">Displaying {{ len .Entries }} entries</span></div>
		<br />
		<table>
			<thead>


@@ 24,12 36,16 @@ var pageTmpl = `<html>
				</tr>
			</thead>
			<tbody>
				{{range .Entries}}
				{{range $index, $entry := .Entries}}
				<tr>
					{{if .URL}}
					<td><a href="{{.URL}}">{{.Name}}</a></td>
					{{else}}
					<td>{{.Name}}</td>
					{{end}}
					<td>{{.Rating}}</td>
					<td>{{join .GradeLevels}}</td>
					<td>{{join .Subjects}}</td>
					<td>{{joinbr .Subjects}}</td>
					<td>{{.Description}}</td>
				</tr>
				{{end}}


@@ 38,15 54,33 @@ var pageTmpl = `<html>
	</body>
</html>`

// Function renders the database as an html table
func Function(w http.ResponseWriter, r *http.Request) {
// WriteHTML writes the given DB to an io.Writer after applying an HTML template
func WriteHTML(w io.Writer, db DB) error {
	t := template.Must(template.New("sb").Funcs(
		template.FuncMap{
			"join": func(v []string) string {
				return strings.Join(v, ",")
			},
			"joinbr": func(v []string) template.HTML {
				return template.HTML(strings.Join(v, "<br />"))
			},
		}).Parse(pageTmpl))
	if err := t.Execute(w, db); err != nil {
		return fmt.Errorf("could not generate page: %v", err)
	}

	return nil
}

// Serve renders the database as an html table
func Serve(w http.ResponseWriter, r *http.Request) {
	db, err := Build()
	if err != nil {
		log.Printf("could not serve page: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
	}

	matcher := NewMatcher() //.Rating(rating).Names(names).Grades(grades).Subjects(subjects)
	matcher := NewMatcher()
	for k, v := range r.URL.Query() {
		switch k {
		case "rating":


@@ 72,21 106,14 @@ func Function(w http.ResponseWriter, r *http.Request) {
		case "subject":
			matcher.Subjects(v)
		}
		// TODO: the rest
	}

	filtered, err := Filter(db, matcher)
	if err != nil {
		log.Printf("could not serve page: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
	if err == nil {
		err = WriteHTML(w, filtered)
	}

	t := template.Must(template.New("sb").Funcs(
		template.FuncMap{"join": func(v []string) string {
			return strings.Join(v, ",")
		},
		}).Parse(pageTmpl))
	if err := t.Execute(w, filtered); err != nil {
	if err != nil {
		log.Printf("could not serve page: %v\n", err)
		w.WriteHeader(http.StatusInternalServerError)
	}

R function_test.go => serve_test.go +2 -2
@@ 6,8 6,8 @@ import (
	"testing"
)

func TestFunction(t *testing.T) {
func TestServe(t *testing.T) {
	t.Skip("Bad to test this!")
	http.HandleFunc("/", Function)
	http.HandleFunc("/", Serve)
	log.Fatal(http.ListenAndServe(":10000", nil))
}