~gjabell/mfn

e88bbe136fa31544f4d7615eabb18590feb2ef8f — Galen Abell 1 year, 10 months ago
Initial commit
5 files changed, 326 insertions(+), 0 deletions(-)

A .gitignore
A README.md
A go.mod
A go.sum
A main.go
A  => .gitignore +3 -0
@@ 1,3 @@
mfw
config.toml
last_entry.txt
\ No newline at end of file

A  => README.md +33 -0
@@ 1,33 @@
# mfw

A small command-line utility to send webhook notifications when [Miniflux](https://miniflux.app) finds new RSS entries.

## Example Usage

1. Create a TOML configuration file with the following format:

```
server = "https://rss.example.com" # the url of the Miniflux instance
username = "myuser" # the username used to login to the Miniflux instance
password = "mypass" # the password used to login to the Miniflux instance
db_path = "/var/lib/mfw/last_entry.txt" # path where the last entry is stored
webhook_template = """
<h3>{{ .Title }}{{ if ne .Author "" }} • <i>{{ .Author }}</i>{{ end }}</h3>
{{ if ne .Content "" }}
<br/>
<blockquote>{{ printf "%.250s" .Content }}{{ if gt (len .Content) 250 }}...{{ end }}</blockquote>
{{ end }}
<br/>
<a href="{{ .URL }}">{{ .URL }}</a>""" # golang html/template-compatible template string, which will be sent as the body of the webhook
webhook_url = "https://webhooks.example.com" # the url of the webhook endpoint
```

2. Ensure `mwf` can successfully send a webhook: `mwf -t -c [path to config]`

3. Run `mwf` once to sync the last entry (no webhooks will be sent during the initial population): `mwf -c [path to config]`

4. Periodically run `mwf` (for example, as a cron job) to send notifications when new entries are picked up by Miniflux: `mwf -c [path to config]`

## Implementation

`mwf` uses the official Miniflux Golang client to query Miniflux for new entries. On the first run, `mwf` will determine the id of the most recent entry and store it in a plaintext file. On subsequent runs, `mwf` will only request entries posted after the latest entry, updating the latest entry id on each run. Each entry requested after the initial sync will trigger a webhook notification to the given webhook endpoint, with the body text parsed from the specified template.

A  => go.mod +10 -0
@@ 1,10 @@
module git.sr.ht/~gjabell/mfw

go 1.13

require (
	git.sr.ht/~sircmpwn/getopt v0.0.0-20190808004552-daaf1274538b
	github.com/BurntSushi/toml v0.3.1
	github.com/mattn/go-sqlite3 v2.0.2+incompatible
	miniflux.app v0.0.0-20200125042750-15727f716a83
)

A  => go.sum +57 -0
@@ 1,57 @@
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
git.sr.ht/~sircmpwn/getopt v0.0.0-20190808004552-daaf1274538b h1:da5JBQ6dcW14aWnEf/pFRIMV2PsqTQEWmR+V2sw5oxU=
git.sr.ht/~sircmpwn/getopt v0.0.0-20190808004552-daaf1274538b/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U=
github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/tdewolff/minify/v2 v2.5.2/go.mod h1:Q6mWHrmspbdRX0ZuUUoKIT8bDjVVXpIJ73ux7p7HZGg=
github.com/tdewolff/parse/v2 v2.3.9/go.mod h1:HansaqmN4I/U7L6/tUp0NcwT2tFO0F4EAWYGSDzkYNk=
github.com/tdewolff/test v1.0.0/go.mod h1:DiQUlutnqlEvdvhSn2LPGy4TFwRauAaYDsL+683RNX4=
github.com/tdewolff/test v1.0.4/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
miniflux.app v0.0.0-20200125042750-15727f716a83 h1:dhLBkGl+VAP+R4iQvE30xqh8xjBFs4ErypWhq4I0dXA=
miniflux.app v0.0.0-20200125042750-15727f716a83/go.mod h1:BqPkLb01QU9bhx4/rOlLomYGfhpHhexBsEsAx8uDpCw=

A  => main.go +223 -0
@@ 1,223 @@
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"git.sr.ht/~sircmpwn/getopt"
	"github.com/BurntSushi/toml"

	mf "miniflux.app/client"
)

var version = "0.1.0"
var webhookTemplate *template.Template

type config struct {
	Server          string `toml:"server"`
	Username        string `toml:"username"`
	Password        string `toml:"password"`
	DbPath          string `toml:"db_path"`
	WebhookTemplate string `toml:"webhook_template"`
	WebhookURL      string `toml:"webhook_url"`
}

func loadConfig(configPath string) (*config, error) {
	var c config

	if _, err := toml.DecodeFile(configPath, &c); err != nil {
		return nil, err
	}

	// validation
	for k, v := range map[string]string{
		"server url":       c.Server,
		"username":         c.Username,
		"password":         c.Password,
		"database path":    c.DbPath,
		"webhook template": c.WebhookTemplate,
		"webhook url":      c.WebhookURL,
	} {
		if v == "" {
			return nil, fmt.Errorf("you must provide a valid %s", k)
		}
	}

	// parse the template
	tmpl, err := template.New("webhook").Parse(c.WebhookTemplate)
	if err != nil {
		return nil, err
	}
	webhookTemplate = tmpl

	return &c, nil
}

func loadLastEntry(filepath string) (int64, error) {
	bytes, err := ioutil.ReadFile(filepath)
	if err != nil {
		if os.IsNotExist(err) {
			// no last entry
			return -1, nil
		}
		return -1, err
	}

	str := strings.TrimSpace(string(bytes))
	id, err := strconv.ParseInt(str, 10, 64)
	if err != nil {
		return -1, err
	}

	return id, nil
}

func saveLastEntry(filepath string, id int64) error {
	return ioutil.WriteFile(filepath, []byte(strconv.FormatInt(id, 10)), 0644)
}

func sendWebhook(entry *mf.Entry, webhookURL string) error {
	var text bytes.Buffer
	if err := webhookTemplate.Execute(&text, entry); err != nil {
		return err
	}

	body := map[string]string{"text": text.String(), "format": "html", "displayName": "Miniflux", "avatarUrl": "https://miniflux.app/favicon.ico"}
	jsonVal, err := json.Marshal(body)
	if err != nil {
		return err
	}

	res, err := http.Post(webhookURL, "application/json", bytes.NewBuffer(jsonVal))
	if err != nil {
		return err
	}
	resBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil
	}
	log.Printf("Webhook response: %s", string(resBytes))

	return nil
}

func testWebhook(webhookURL string) error {
	e := &mf.Entry{
		ID:         0,
		UserID:     0,
		FeedID:     0,
		Status:     "unread",
		Hash:       "deadbeef",
		Title:      "Test Entry",
		URL:        "https://miniflux.app",
		Date:       time.Now(),
		Content:    "This is a test miniflux entry",
		Author:     "Author",
		Starred:    false,
		Enclosures: make([]*mf.Enclosure, 0),
		Feed:       nil,
	}

	return sendWebhook(e, webhookURL)
}

func main() {
	showHelp := getopt.Bool("h", false, "show usage")
	webhookTest := getopt.Bool("t", false, "test the webhook endpoint")
	showVersion := getopt.Bool("v", false, "show version")
	var configPath string
	getopt.StringVar(&configPath, "c", "", "configuration file path")

	err := getopt.Parse()
	if err != nil {
		log.Fatalf("Failed to parse args: %s", err)
	}

	if *showHelp {
		fmt.Println("Usage: mfw [-v] [-t] -c config")
		os.Exit(0)
	}

	if *showVersion {
		fmt.Printf("mfw %s\n", version)
		os.Exit(0)
	}

	if configPath == "" {
		log.Fatal("You must specify a config location")
	}

	conf, err := loadConfig(configPath)
	if err != nil {
		log.Fatalf("Failed to load config: %s\n", err)
	}

	if *webhookTest {
		log.Println("Sending test webhook...")
		if err := testWebhook(conf.WebhookURL); err != nil {
			log.Fatalf("Test webhook failed: %s\n", err)
		}
		os.Exit(0)
	}

	client := mf.New(conf.Server, conf.Username, conf.Password)

	lastEntry, err := loadLastEntry(conf.DbPath)
	if err != nil {
		log.Fatalf("Failed to load last entry: %s\n", err)
	}

	var notify bool
	filter := &mf.Filter{Order: "id"}
	if lastEntry == -1 {
		// this is the first run, since we don't have a last entry
		// in this case, just request the single most recent entry
		// this will be the entry with the largest ID
		filter.Limit = 1
		filter.Direction = "desc"
		// don't send notifications on the first run
		notify = false
	} else {
		// if we already have entries, request all entries since the
		// last entry ID
		filter.Limit = 0
		filter.Direction = "asc"
		filter.AfterEntryID = lastEntry
		notify = true
	}

	newEntriesSet, err := client.Entries(filter)
	if err != nil {
		log.Fatalf("Failed to fetch entries: %s\n", err)
	}

	newEntries := newEntriesSet.Entries
	for _, e := range newEntries {
		log.Printf("New entry: %d\n", e.ID)

		if e.ID > lastEntry {
			lastEntry = e.ID
		}

		if !notify {
			continue
		}

		if err := sendWebhook(e, conf.WebhookURL); err != nil {
			log.Printf("Failed to send webhook: %s\n", err)
		}
	}
	log.Printf("Saving new last entry: %d\n", lastEntry)
	if err := saveLastEntry(conf.DbPath, lastEntry); err != nil {
		log.Fatalf("Failed to save new entries: %s\n", err)
	}
}