~emersion/soju

5b4469fcb7f074bbb72e7970094abfee9cdfa037 — Simon Ser 19 days ago 5e11e71
Use BARE for internal message IDs

This allows to have shorter and more future-proof IDs. This also
guarantees the IDs will only use reasonable ASCII characters (no
spaces), removing the need to encode them for PING/PONG tokens.
6 files changed, 132 insertions(+), 55 deletions(-)

M downstream.go
M go.mod
M go.sum
M msgstore.go
M msgstore_fs.go
M msgstore_memory.go
M downstream.go => downstream.go +3 -10
@@ 350,7 350,7 @@ func (dc *downstreamConn) advanceMessageWithID(msg *irc.Message, id string) {

// ackMsgID acknowledges that a message has been received.
func (dc *downstreamConn) ackMsgID(id string) {
	netID, entity, _, err := parseMsgID(id)
	netID, entity, err := parseMsgID(id, nil)
	if err != nil {
		dc.logger.Printf("failed to ACK message ID %q: %v", id, err)
		return


@@ 365,7 365,7 @@ func (dc *downstreamConn) ackMsgID(id string) {
}

func (dc *downstreamConn) sendPing(msgID string) {
	token := "soju-msgid-" + base64.RawURLEncoding.EncodeToString([]byte(msgID))
	token := "soju-msgid-" + msgID
	dc.SendMessage(&irc.Message{
		Command: "PING",
		Params:  []string{token},


@@ 377,14 377,7 @@ func (dc *downstreamConn) handlePong(token string) {
		dc.logger.Printf("received unrecognized PONG token %q", token)
		return
	}
	token = strings.TrimPrefix(token, "soju-msgid-")
	b, err := base64.RawURLEncoding.DecodeString(token)
	if err != nil {
		dc.logger.Printf("received malformed PONG token: %v", err)
		return
	}
	msgID := string(b)

	msgID := strings.TrimPrefix(token, "soju-msgid-")
	dc.ackMsgID(msgID)
}


M go.mod => go.mod +1 -0
@@ 4,6 4,7 @@ go 1.13

require (
	git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc
	git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9
	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21
	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
	github.com/klauspost/compress v1.11.7 // indirect

M go.sum => go.sum +7 -4
@@ 1,6 1,8 @@
git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc h1:51BD67xFX+bozd3ZRuOUfalrhx4/nQSh6A9lI08rYOk=
git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc/go.mod h1:t+Ww6SR24yYnXzEWiNlOY0AFo5E9B73X++10lrSpp4U=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw=
git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9 h1:GpgMhmmlPgaKOSU3WnoaSpZGWgnprcS+ss2w9SchYu4=
git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=


@@ 36,7 38,6 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=


@@ 56,8 57,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=


@@ 72,7 74,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY=
golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=


@@ 89,5 90,7 @@ gopkg.in/irc.v3 v3.1.4/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=

M msgstore.go => msgstore.go +67 -11
@@ 1,11 1,12 @@
package soju

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"strconv"
	"strings"
	"time"

	"git.sr.ht/~sircmpwn/go-bare"
	"gopkg.in/irc.v3"
)



@@ 29,18 30,73 @@ type chatHistoryMessageStore interface {
	LoadAfterTime(network *network, entity string, t time.Time, limit int) ([]*irc.Message, error)
}

func formatMsgID(netID int64, entity, extra string) string {
	return fmt.Sprintf("%v %v %v", netID, entity, extra)
type msgIDType uint

const (
	msgIDNone msgIDType = iota
	msgIDMemory
	msgIDFS
)

const msgIDVersion uint = 0

type msgIDHeader struct {
	Version uint
	Network bare.Int
	Target  string
	Type    msgIDType
}

type msgIDBody interface {
	msgIDType() msgIDType
}

func parseMsgID(s string) (netID int64, entity, extra string, err error) {
	l := strings.SplitN(s, " ", 3)
	if len(l) != 3 {
		return 0, "", "", fmt.Errorf("invalid message ID %q: expected 3 fields", s)
func formatMsgID(netID int64, target string, body msgIDBody) string {
	var buf bytes.Buffer
	w := bare.NewWriter(&buf)

	header := msgIDHeader{
		Version: msgIDVersion,
		Network: bare.Int(netID),
		Target:  target,
		Type:    body.msgIDType(),
	}
	if err := bare.MarshalWriter(w, &header); err != nil {
		panic(err)
	}
	netID, err = strconv.ParseInt(l[0], 10, 64)
	if err := bare.MarshalWriter(w, body); err != nil {
		panic(err)
	}
	return base64.RawURLEncoding.EncodeToString(buf.Bytes())
}

func parseMsgID(s string, body msgIDBody) (netID int64, target string, err error) {
	b, err := base64.RawURLEncoding.DecodeString(s)
	if err != nil {
		return 0, "", "", fmt.Errorf("invalid message ID %q: %v", s, err)
		return 0, "", fmt.Errorf("invalid internal message ID: %v", err)
	}

	r := bare.NewReader(bytes.NewReader(b))

	var header msgIDHeader
	if err := bare.UnmarshalBareReader(r, &header); err != nil {
		return 0, "", fmt.Errorf("invalid internal message ID: %v", err)
	}

	if header.Version != msgIDVersion {
		return 0, "", fmt.Errorf("invalid internal message ID: got version %v, want %v", header.Version, msgIDVersion)
	}
	return netID, l[1], l[2], nil

	if body != nil {
		typ := body.msgIDType()
		if header.Type != typ {
			return 0, "", fmt.Errorf("invalid internal message ID: got type %v, want %v", header.Type, typ)
		}

		if err := bare.UnmarshalBareReader(r, body); err != nil {
			return 0, "", fmt.Errorf("invalid internal message ID: %v", err)
		}
	}

	return int64(header.Network), header.Target, nil
}

M msgstore_fs.go => msgstore_fs.go +40 -21
@@ 9,6 9,7 @@ import (
	"strings"
	"time"

	"git.sr.ht/~sircmpwn/go-bare"
	"gopkg.in/irc.v3"
)



@@ 16,6 17,45 @@ const fsMessageStoreMaxTries = 100

var escapeFilename = strings.NewReplacer("/", "-", "\\", "-")

type date struct {
	Year, Month, Day int
}

func newDate(t time.Time) date {
	year, month, day := t.Date()
	return date{year, int(month), day}
}

func (d date) Time() time.Time {
	return time.Date(d.Year, time.Month(d.Month), d.Day, 0, 0, 0, 0, time.Local)
}

type fsMsgID struct {
	Date   date
	Offset bare.Int
}

func (fsMsgID) msgIDType() msgIDType {
	return msgIDFS
}

func parseFSMsgID(s string) (netID int64, entity string, t time.Time, offset int64, err error) {
	var id fsMsgID
	netID, entity, err = parseMsgID(s, &id)
	if err != nil {
		return 0, "", time.Time{}, 0, err
	}
	return netID, entity, id.Date.Time(), int64(id.Offset), nil
}

func formatFSMsgID(netID int64, entity string, t time.Time, offset int64) string {
	id := fsMsgID{
		Date:   newDate(t),
		Offset: bare.Int(offset),
	}
	return formatMsgID(netID, entity, &id)
}

// fsMessageStore is a per-user on-disk store for IRC messages.
type fsMessageStore struct {
	root string


@@ 36,27 76,6 @@ func (ms *fsMessageStore) logPath(network *network, entity string, t time.Time) 
	return filepath.Join(ms.root, escapeFilename.Replace(network.GetName()), escapeFilename.Replace(entity), filename)
}

func parseFSMsgID(s string) (netID int64, entity string, t time.Time, offset int64, err error) {
	netID, entity, extra, err := parseMsgID(s)
	if err != nil {
		return 0, "", time.Time{}, 0, err
	}

	var year, month, day int
	_, err = fmt.Sscanf(extra, "%04d-%02d-%02d %d", &year, &month, &day, &offset)
	if err != nil {
		return 0, "", time.Time{}, 0, fmt.Errorf("invalid message ID %q: %v", s, err)
	}
	t = time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
	return netID, entity, t, offset, nil
}

func formatFSMsgID(netID int64, entity string, t time.Time, offset int64) string {
	year, month, day := t.Date()
	extra := fmt.Sprintf("%04d-%02d-%02d %d", year, month, day, offset)
	return formatMsgID(netID, entity, extra)
}

// nextMsgID queries the message ID for the next message to be written to f.
func nextFSMsgID(network *network, entity string, t time.Time, f *os.File) (string, error) {
	offset, err := f.Seek(0, io.SeekEnd)

M msgstore_memory.go => msgstore_memory.go +14 -9
@@ 2,29 2,34 @@ package soju

import (
	"fmt"
	"strconv"
	"time"

	"git.sr.ht/~sircmpwn/go-bare"
	"gopkg.in/irc.v3"
)

const messageRingBufferCap = 4096

type memoryMsgID struct {
	Seq bare.Uint
}

func (memoryMsgID) msgIDType() msgIDType {
	return msgIDMemory
}

func parseMemoryMsgID(s string) (netID int64, entity string, seq uint64, err error) {
	netID, entity, extra, err := parseMsgID(s)
	var id memoryMsgID
	netID, entity, err = parseMsgID(s, &id)
	if err != nil {
		return 0, "", 0, err
	}
	seq, err = strconv.ParseUint(extra, 10, 64)
	if err != nil {
		return 0, "", 0, fmt.Errorf("failed to parse message ID %q: %v", s, err)
	}
	return netID, entity, seq, nil
	return netID, entity, uint64(id.Seq), nil
}

func formatMemoryMsgID(netID int64, entity string, seq uint64) string {
	extra := strconv.FormatUint(seq, 10)
	return formatMsgID(netID, entity, extra)
	id := memoryMsgID{bare.Uint(seq)}
	return formatMsgID(netID, entity, &id)
}

type ringBufferKey struct {