M cmd/searchix-web/main.go => cmd/searchix-web/main.go +20 -16
@@ 4,8 4,6 @@ import (
"context"
"flag"
"fmt"
- "log"
- "log/slog"
"os"
"os/signal"
@@ 13,6 11,7 @@ import (
"go.alanpearce.eu/searchix"
"go.alanpearce.eu/searchix/internal/config"
+ "go.alanpearce.eu/x/log"
)
var (
@@ 22,10 21,10 @@ var (
false,
"print default configuration and exit",
)
- liveReload = flag.Bool("live", false, "whether to enable live reloading (development)")
- replace = flag.Bool("replace", false, "replace existing index and exit")
- update = flag.Bool("update", false, "update index and exit")
- version = flag.Bool("version", false, "print version information")
+ dev = flag.Bool("dev", false, "enable live reloading and nicer logging")
+ replace = flag.Bool("replace", false, "replace existing index and exit")
+ update = flag.Bool("update", false, "update index and exit")
+ version = flag.Bool("version", false, "print version information")
)
func main() {
@@ 45,23 44,28 @@ func main() {
os.Exit(0)
}
- cfg, err := config.GetConfig(*configFile)
+ logger := log.Configure(!*dev)
+
+ cfg, err := config.GetConfig(*configFile, logger)
if err != nil {
- // only use log functions after the config file has been read successfully
- log.Fatalf("Failed to parse config file: %v", err)
+ logger.Fatal("Failed to parse config file", "error", err)
}
- s, err := searchix.New(cfg)
+
+ log.SetLevel(cfg.LogLevel)
+
+ s, err := searchix.New(cfg, logger)
if err != nil {
- log.Fatalf("Failed to initialise searchix: %v", err)
+ logger.Fatal("Failed to initialise searchix", "error", err)
}
err = s.SetupIndex(&searchix.IndexOptions{
Update: *update,
Replace: *replace,
LowMemory: cfg.Importer.LowMemory,
+ Logger: logger,
})
if err != nil {
- log.Fatalf("Failed to setup index: %v", err)
+ logger.Fatal("Failed to setup index", "error", err)
}
if *replace || *update {
@@ 72,15 76,15 @@ func main() {
defer cancel()
go func() {
- err = s.Start(ctx, *liveReload)
+ err = s.Start(ctx, *dev)
if err != nil {
// Error starting or closing listener:
- log.Fatalf("error: %v", err)
+ logger.Fatal("error", "error", err)
}
}()
<-ctx.Done()
- slog.Debug("calling stop")
+ logger.Debug("calling stop")
s.Stop()
- slog.Debug("done")
+ logger.Debug("done")
}
M defaults.toml => defaults.toml +2 -2
@@ 1,7 1,7 @@
# Path to store index data.
DataPath = './data'
-# How much information to log, one of 'debug', 'info', 'warn', 'error'.
-LogLevel = 'INFO'
+# How much information to log, one of 'debug', 'info', 'warn', 'error', 'panic', 'fatal'.
+LogLevel = 'info'
# Settings for the web server
[Web]
M go.mod => go.mod +8 -2
@@ 1,6 1,6 @@
module go.alanpearce.eu/searchix
-go 1.22.2
+go 1.22.3
require (
badc0de.net/pkg/flagutil v1.0.1
@@ 16,13 16,15 @@ require (
github.com/osdevisnot/sorvor v0.4.4
github.com/pelletier/go-toml/v2 v2.2.2
github.com/pkg/errors v0.9.1
- github.com/shengyanli1982/law v0.1.16
github.com/stoewer/go-strcase v1.3.0
github.com/yuin/goldmark v1.7.2
+ go.alanpearce.eu/x v0.0.0-20240701200753-a70ddb349b02
+ go.uber.org/zap v1.27.0
golang.org/x/net v0.26.0
)
require (
+ github.com/Code-Hex/dd v1.1.0 // indirect
github.com/RoaringBitmap/roaring v1.9.4 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/blevesearch/geo v0.1.20 // indirect
@@ 48,8 50,12 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect
+ github.com/sykesm/zap-logfmt v0.0.4 // indirect
+ github.com/thessem/zap-prettyconsole v0.5.0 // indirect
go.etcd.io/bbolt v1.3.10 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
+ moul.io/zapfilter v1.7.0 // indirect
)
M go.sum => go.sum +84 -2
@@ 1,5 1,8 @@
badc0de.net/pkg/flagutil v1.0.1 h1:0ZgBzd3FehDUA8DJ70/phsnDH61/3aYMyx8Wd84KqQo=
badc0de.net/pkg/flagutil v1.0.1/go.mod h1:HwwkfbImu+u288bnLaYDGqBxkJzvqi5YzKofmgkMLvk=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Code-Hex/dd v1.1.0 h1:VEtTThnS9l7WhpKUIpdcWaf0B8Vp0LeeSEsxA1DZseI=
+github.com/Code-Hex/dd v1.1.0/go.mod h1:VaMyo/YjTJ3d4qm/bgtrUkT2w+aYwJ07Y7eCWyrJr1w=
github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ=
github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
github.com/a-h/templ v0.2.707 h1:T1Gkd2ugbRglZ9rYw/VBchWOSZVKmetDbBkm4YubM7U=
@@ 8,6 11,7 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck=
github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ=
+github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
@@ 68,8 72,13 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ 85,12 94,14 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/shengyanli1982/law v0.1.16 h1:sQykz7ysBxYZSHkDdWj9C5EOE1Fez/PYg1bxij49Omg=
-github.com/shengyanli1982/law v0.1.16/go.mod h1:20k9YnOTwilUB4X5Z4S7TIX5Ek1Ok4xfx8V8ZxIWlyM=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ 98,6 109,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
@@ 105,24 117,94 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI=
+github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA=
+github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
+github.com/thessem/zap-prettyconsole v0.5.0 h1:AOu1GGUuDkGmj4tgRPSVf0vYGzDM+6cPWjKOcmjEcQs=
+github.com/thessem/zap-prettyconsole v0.5.0/go.mod h1:3qfsE7y+bLOq7EQ+fMZHD3HYEp24ULFf5nhLSx6rjrE=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.7.2 h1:NjGd7lO7zrUn/A7eKwn5PEOt4ONYGqpxSEeZuduvgxc=
github.com/yuin/goldmark v1.7.2/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+go.alanpearce.eu/x v0.0.0-20240701200753-a70ddb349b02 h1:Ed0aWwSR9+Z7k/6LnG8iDXTW3Sb48Ahanjy7i83aboU=
+go.alanpearce.eu/x v0.0.0-20240701200753-a70ddb349b02/go.mod h1:GaYgUfXSlaHBvdrInLYyKDMKo2Bmx1+IIFrlnZkZW+A=
go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
+go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
+go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
+go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+moul.io/zapfilter v1.7.0 h1:7aFrG4N72bDH9a2BtYUuUaDS981Dxu3qybWfeqaeBDU=
+moul.io/zapfilter v1.7.0/go.mod h1:M+N2s+qZiA+bzRoyKMVRxyuERijS2ovi2pnMyiOGMvc=
M gomod2nix.toml => gomod2nix.toml +21 -3
@@ 4,6 4,9 @@ schema = 3
[mod."badc0de.net/pkg/flagutil"]
version = "v1.0.1"
hash = "sha256-0LRWL5DUHW3gXQhPAhUCxnUCN7HN1qKI2yZp8MrDN6M="
+ [mod."github.com/Code-Hex/dd"]
+ version = "v1.1.0"
+ hash = "sha256-9aoekzjMXuJmR0/7bfu4y3SfcWBgdfYybB7gt4sNKfk="
[mod."github.com/RoaringBitmap/roaring"]
version = "v1.9.4"
hash = "sha256-OKOLQ/PsH6630Vb5/9yG28TLIPGxdG9WDbAZxgK8EcI="
@@ 115,18 118,30 @@ schema = 3
[mod."github.com/pkg/errors"]
version = "v0.9.1"
hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw="
- [mod."github.com/shengyanli1982/law"]
- version = "v0.1.16"
- hash = "sha256-UsO5qqKiREvwlz3JDKFAJFmXEu3JHYZOXibGgcgPNGY="
[mod."github.com/stoewer/go-strcase"]
version = "v1.3.0"
hash = "sha256-X0ilcefeqVQ44B9WT6euCMcigs7oLFypOQaGI33kGr8="
+ [mod."github.com/sykesm/zap-logfmt"]
+ version = "v0.0.4"
+ hash = "sha256-KXVFtOU54chusK8AhZrzrvbbNmzq1mNrhs/7OmO+huE="
+ [mod."github.com/thessem/zap-prettyconsole"]
+ version = "v0.5.0"
+ hash = "sha256-bOhManZjabZYHZwsaobaM9aPW+sUeqIfV+UnQLMaz54="
[mod."github.com/yuin/goldmark"]
version = "v1.7.2"
hash = "sha256-0rjUJP5WJy6227Epkgm/UHU9xzvrOAvYW+Y3EC+MkTE="
+ [mod."go.alanpearce.eu/x"]
+ version = "v0.0.0-20240701200753-a70ddb349b02"
+ hash = "sha256-TRQgdPye/Q9LiM1XCDgxNrHTZKtSzuJ7lbNbWjkZvU4="
[mod."go.etcd.io/bbolt"]
version = "v1.3.10"
hash = "sha256-uEnz6jmmgT+hlwdZ8ns5NCJSbZcB4i123FF2cn2CbQA="
+ [mod."go.uber.org/multierr"]
+ version = "v1.11.0"
+ hash = "sha256-Lb6rHHfR62Ozg2j2JZy3MKOMKdsfzd1IYTR57r3Mhp0="
+ [mod."go.uber.org/zap"]
+ version = "v1.27.0"
+ hash = "sha256-8655KDrulc4Das3VRduO9MjCn8ZYD5WkULjCvruaYsU="
[mod."golang.org/x/net"]
version = "v0.26.0"
hash = "sha256-WfY33QERNbcIiDkH3+p2XGrAVqvWBQfc8neUt6TH6dQ="
@@ 139,3 154,6 @@ schema = 3
[mod."google.golang.org/protobuf"]
version = "v1.34.2"
hash = "sha256-nMTlrDEE2dbpWz50eQMPBQXCyQh4IdjrTIccaU0F3m0="
+ [mod."moul.io/zapfilter"]
+ version = "v1.7.0"
+ hash = "sha256-H6j5h8w123Y7d0zvKGkL5jiRYICtjmgzd2P/eeNaLrs="
M internal/components/results.templ => internal/components/results.templ +1 -6
@@ 2,16 2,11 @@ package components
import (
"strconv"
- "log/slog"
"go.alanpearce.eu/searchix/internal/nix"
)
func convertMatch[I nix.Importable](m nix.Importable) *I {
- i, ok := m.(I)
- if !ok {
- slog.Warn("Converting match failed", "match", m)
- return nil
- }
+ i := m.(I)
return &i
}
M internal/config/config.go => internal/config/config.go +3 -3
@@ 1,7 1,6 @@
package config
import (
- "log/slog"
"maps"
"net/url"
"os"
@@ 9,6 8,7 @@ import (
"github.com/pelletier/go-toml/v2"
"github.com/pkg/errors"
+ "go.alanpearce.eu/x/log"
)
var Version string
@@ 102,10 102,10 @@ func mustLocalTime(in string) (time LocalTime) {
return
}
-func GetConfig(filename string) (*Config, error) {
+func GetConfig(filename string, log *log.Logger) (*Config, error) {
config := DefaultConfig
if filename != "" {
- slog.Debug("reading config", "filename", filename)
+ log.Debug("reading config", "filename", filename)
f, err := os.Open(filename)
if err != nil {
return nil, errors.Wrap(err, "reading config failed")
M internal/config/structs.go => internal/config/structs.go +6 -5
@@ 5,14 5,15 @@ package config
import (
"fmt"
- "log/slog"
+
+ "go.uber.org/zap/zapcore"
)
type Config struct {
- DataPath string `comment:"Path to store index data."`
- LogLevel slog.Level `comment:"How much information to log, one of 'debug', 'info', 'warn', 'error'."`
- Web *Web `comment:"Settings for the web server"`
- Importer *Importer `comment:"Settings for the import job"`
+ DataPath string `comment:"Path to store index data."`
+ LogLevel zapcore.Level `comment:"How much information to log, one of 'debug', 'info', 'warn', 'error', 'panic', 'fatal'."`
+ Web *Web `comment:"Settings for the web server"`
+ Importer *Importer `comment:"Settings for the import job"`
}
type Web struct {
M internal/fetcher/channel.go => internal/fetcher/channel.go +3 -3
@@ 3,7 3,6 @@ package fetcher
import (
"context"
"fmt"
- "log/slog"
"os"
"os/exec"
"path"
@@ 13,6 12,7 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/index"
+ "go.alanpearce.eu/x/log"
"github.com/pkg/errors"
)
@@ 20,12 20,12 @@ import (
type ChannelFetcher struct {
Source *config.Source
SourceFile string
- Logger *slog.Logger
+ Logger *log.Logger
}
func NewChannelFetcher(
source *config.Source,
- logger *slog.Logger,
+ logger *log.Logger,
) (*ChannelFetcher, error) {
switch source.Importer {
case config.Options:
M internal/fetcher/download.go => internal/fetcher/download.go +4 -4
@@ 3,24 3,24 @@ package fetcher
import (
"context"
"fmt"
- "log/slog"
"net/url"
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/index"
"github.com/pkg/errors"
+ "go.alanpearce.eu/x/log"
)
type DownloadFetcher struct {
Source *config.Source
SourceFile string
- Logger *slog.Logger
+ Logger *log.Logger
}
func NewDownloadFetcher(
source *config.Source,
- logger *slog.Logger,
+ logger *log.Logger,
) (*DownloadFetcher, error) {
switch source.Importer {
case config.Options:
@@ 59,7 59,7 @@ func (i *DownloadFetcher) FetchIfNeeded(
i.Logger.Debug("preparing to fetch URL", "url", fetchURL)
- body, mtime, err := fetchFileIfNeeded(ctx, sourceUpdated, fetchURL)
+ body, mtime, err := fetchFileIfNeeded(ctx, i.Logger, sourceUpdated, fetchURL)
if err != nil {
i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err)
M internal/fetcher/http.go => internal/fetcher/http.go +4 -3
@@ 4,7 4,6 @@ import (
"context"
"fmt"
"io"
- "log/slog"
"net/http"
"strings"
"time"
@@ 13,6 12,7 @@ import (
"github.com/andybalholm/brotli"
"github.com/pkg/errors"
+ "go.alanpearce.eu/x/log"
)
type brotliReadCloser struct {
@@ 33,6 33,7 @@ func (r *brotliReadCloser) Close() error {
func fetchFileIfNeeded(
ctx context.Context,
+ log *log.Logger,
mtime time.Time,
url string,
) (body io.ReadCloser, newMtime time.Time, err error) {
@@ 68,7 69,7 @@ func fetchFileIfNeeded(
case http.StatusOK:
newMtime, err = time.Parse(time.RFC1123, res.Header.Get("Last-Modified"))
if err != nil {
- slog.Warn(
+ log.Warn(
"could not parse Last-Modified header from response",
"value",
res.Header.Get("Last-Modified"),
@@ 78,7 79,7 @@ func fetchFileIfNeeded(
switch ce := res.Header.Get("Content-Encoding"); ce {
case "br":
- slog.Debug("using brotli encoding")
+ log.Debug("using brotli encoding")
body = newBrotliReader(res.Body)
case "", "identity", "gzip":
body = res.Body
M internal/fetcher/main.go => internal/fetcher/main.go +2 -2
@@ 3,10 3,10 @@ package fetcher
import (
"context"
"io"
- "log/slog"
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/index"
+ "go.alanpearce.eu/x/log"
"github.com/pkg/errors"
)
@@ 23,7 23,7 @@ type Fetcher interface {
func New(
source *config.Source,
- logger *slog.Logger,
+ logger *log.Logger,
) (fetcher Fetcher, err error) {
switch source.Fetcher {
case config.ChannelNixpkgs:
M internal/fetcher/nixpkgs-channel.go => internal/fetcher/nixpkgs-channel.go +4 -4
@@ 3,18 3,18 @@ package fetcher
import (
"context"
"fmt"
- "log/slog"
"net/url"
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/index"
"github.com/pkg/errors"
+ "go.alanpearce.eu/x/log"
)
type NixpkgsChannelFetcher struct {
Source *config.Source
- Logger *slog.Logger
+ Logger *log.Logger
}
func makeChannelURL(channel string, subPath string) (string, error) {
@@ 25,7 25,7 @@ func makeChannelURL(channel string, subPath string) (string, error) {
func NewNixpkgsChannelFetcher(
source *config.Source,
- logger *slog.Logger,
+ logger *log.Logger,
) (*NixpkgsChannelFetcher, error) {
switch source.Importer {
case config.Options, config.Packages:
@@ 66,7 66,7 @@ func (i *NixpkgsChannelFetcher) FetchIfNeeded(
}
i.Logger.Debug("attempting to fetch file", "url", fetchURL)
- body, mtime, err := fetchFileIfNeeded(ctx, sourceMeta.Updated, fetchURL)
+ body, mtime, err := fetchFileIfNeeded(ctx, i.Logger, sourceMeta.Updated, fetchURL)
if err != nil {
i.Logger.Warn("failed to fetch file", "url", fetchURL, "error", err)
M internal/importer/importer.go => internal/importer/importer.go +2 -2
@@ 2,11 2,11 @@ package importer
import (
"context"
- "log/slog"
"sync"
"go.alanpearce.eu/searchix/internal/index"
"go.alanpearce.eu/searchix/internal/nix"
+ "go.alanpearce.eu/x/log"
)
type Importer interface {
@@ 21,7 21,7 @@ func process(
ctx context.Context,
indexer *index.WriteIndex,
processor Processor,
- logger *slog.Logger,
+ logger *log.Logger,
) (bool, error) {
wg := sync.WaitGroup{}
M internal/importer/main.go => internal/importer/main.go +18 -8
@@ 3,7 3,6 @@ package importer
import (
"context"
"fmt"
- "log/slog"
"os/exec"
"slices"
"strings"
@@ 12,18 11,20 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/fetcher"
"go.alanpearce.eu/searchix/internal/index"
+ "go.alanpearce.eu/x/log"
"github.com/pkg/errors"
)
func createSourceImporter(
parent context.Context,
+ log *log.Logger,
meta *index.Meta,
indexer *index.WriteIndex,
forceUpdate bool,
) func(*config.Source) error {
return func(source *config.Source) error {
- logger := slog.With(
+ logger := log.With(
"name",
source.Key,
"fetcher",
@@ 94,9 95,17 @@ func createSourceImporter(
switch source.Importer {
case config.Options:
logger.Debug("processor created", "file", fmt.Sprintf("%T", files.Options))
- processor, err = NewOptionProcessor(files.Options, source)
+ processor, err = NewOptionProcessor(
+ files.Options,
+ source,
+ logger.Named("processor"),
+ )
case config.Packages:
- processor, err = NewPackageProcessor(files.Packages, source)
+ processor, err = NewPackageProcessor(
+ files.Packages,
+ source,
+ logger.Named("processor"),
+ )
}
if err != nil {
return errors.WithMessagef(err, "failed to create processor")
@@ 123,17 132,18 @@ func createSourceImporter(
func Start(
cfg *config.Config,
+ log *log.Logger,
indexer *index.WriteIndex,
forceUpdate bool,
onlyUpdateSources *[]string,
) error {
if len(cfg.Importer.Sources) == 0 {
- slog.Info("No sources enabled")
+ log.Info("No sources enabled")
return nil
}
- slog.Debug("starting importer", "timeout", cfg.Importer.Timeout.Duration)
+ log.Debug("starting importer", "timeout", cfg.Importer.Timeout.Duration)
importCtx, cancelImport := context.WithTimeout(
context.Background(),
cfg.Importer.Timeout.Duration,
@@ 144,7 154,7 @@ func Start(
meta := indexer.Meta
- importSource := createSourceImporter(importCtx, meta, indexer, forceUpdate)
+ importSource := createSourceImporter(importCtx, log, meta, indexer, forceUpdate)
for name, source := range cfg.Importer.Sources {
if onlyUpdateSources != nil && len(*onlyUpdateSources) > 0 {
if !slices.Contains(*onlyUpdateSources, name) {
@@ 153,7 163,7 @@ func Start(
}
err := importSource(source)
if err != nil {
- slog.Error("import failed", "source", name, "error", err)
+ log.Error("import failed", "source", name, "error", err)
}
}
M internal/importer/main_test.go => internal/importer/main_test.go +4 -3
@@ 1,26 1,27 @@
package importer
import (
- "log/slog"
"testing"
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/index"
+ "go.alanpearce.eu/x/log"
)
var cfg = config.DefaultConfig
func BenchmarkImporterLowMemory(b *testing.B) {
tmp := b.TempDir()
- cfg.LogLevel = slog.LevelDebug
+ logger := log.Configure(false)
_, write, _, err := index.OpenOrCreate(tmp, false, &index.Options{
LowMemory: true,
+ Logger: logger.Named("index"),
})
if err != nil {
b.Fatal(err)
}
- err = Start(&cfg, write, false, &[]string{"nixpkgs"})
+ err = Start(&cfg, logger.Named("importer"), write, false, &[]string{"nixpkgs"})
if err != nil {
b.Fatal(err)
}
M internal/importer/options.go => internal/importer/options.go +13 -7
@@ 3,11 3,11 @@ package importer
import (
"context"
"io"
- "log/slog"
"reflect"
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/nix"
+ "go.alanpearce.eu/x/log"
"github.com/bcicen/jstream"
"github.com/mitchellh/mapstructure"
@@ 35,7 35,7 @@ type nixOptionJSON struct {
Type string
}
-func convertValue(nj *nixValueJSON) *nix.Value {
+func (i *OptionIngester) convertValue(nj *nixValueJSON) *nix.Value {
if nj == nil {
return nil
}
@@ 49,7 49,7 @@ func convertValue(nj *nixValueJSON) *nix.Value {
Markdown: nix.Markdown(nj.Text),
}
default:
- slog.Warn("got unexpected Value type", "type", nj.Type, "text", nj.Text)
+ i.log.Warn("got unexpected Value type", "type", nj.Type, "text", nj.Text)
return nil
}
@@ 58,14 58,20 @@ func convertValue(nj *nixValueJSON) *nix.Value {
type OptionIngester struct {
dec *jstream.Decoder
ms *mapstructure.Decoder
+ log *log.Logger
optJSON nixOptionJSON
infile io.ReadCloser
source *config.Source
}
-func NewOptionProcessor(infile io.ReadCloser, source *config.Source) (*OptionIngester, error) {
+func NewOptionProcessor(
+ infile io.ReadCloser,
+ source *config.Source,
+ log *log.Logger,
+) (*OptionIngester, error) {
i := OptionIngester{
dec: jstream.NewDecoder(infile, 1).EmitKV(),
+ log: log,
optJSON: nixOptionJSON{},
infile: infile,
source: source,
@@ 163,14 169,14 @@ func (i *OptionIngester) Process(ctx context.Context) (<-chan nix.Importable, <-
decs[i] = nix.Link(d)
}
- // slog.Debug("sending option", "name", kv.Key)
+ // log.Debug("sending option", "name", kv.Key)
results <- nix.Option{
Name: kv.Key,
Source: i.source.Key,
Declarations: decs,
- Default: convertValue(i.optJSON.Default),
+ Default: i.convertValue(i.optJSON.Default),
Description: nix.Markdown(i.optJSON.Description),
- Example: convertValue(i.optJSON.Example),
+ Example: i.convertValue(i.optJSON.Example),
RelatedPackages: nix.Markdown(i.optJSON.RelatedPackages),
Loc: i.optJSON.Loc,
Type: i.optJSON.Type,
M internal/importer/package.go => internal/importer/package.go +8 -1
@@ 9,6 9,7 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/nix"
+ "go.alanpearce.eu/x/log"
"github.com/bcicen/jstream"
"github.com/mitchellh/mapstructure"
@@ 40,6 41,7 @@ type maintainerJSON struct {
type PackageIngester struct {
dec *jstream.Decoder
ms *mapstructure.Decoder
+ log *log.Logger
pkg packageJSON
infile io.ReadCloser
source *config.Source
@@ 60,9 62,14 @@ func makeAdhocPlatform(v any) string {
return string(s)
}
-func NewPackageProcessor(infile io.ReadCloser, source *config.Source) (*PackageIngester, error) {
+func NewPackageProcessor(
+ infile io.ReadCloser,
+ source *config.Source,
+ log *log.Logger,
+) (*PackageIngester, error) {
i := &PackageIngester{
dec: jstream.NewDecoder(infile, 2).EmitKV(),
+ log: log,
pkg: packageJSON{},
infile: infile,
source: source,
M internal/index/index_meta.go => internal/index/index_meta.go +9 -6
@@ 2,11 2,11 @@ package index
import (
"encoding/json"
- "log/slog"
"os"
"time"
"go.alanpearce.eu/searchix/internal/file"
+ "go.alanpearce.eu/x/log"
"github.com/pkg/errors"
)
@@ 26,10 26,11 @@ type data struct {
type Meta struct {
path string
+ log *log.Logger
data
}
-func createMeta(path string) (*Meta, error) {
+func createMeta(path string, log *log.Logger) (*Meta, error) {
exists, err := file.Exists(path)
if err != nil {
return nil, errors.WithMessage(err, "could not check for existence of index metadata")
@@ 40,19 41,20 @@ func createMeta(path string) (*Meta, error) {
return &Meta{
path: path,
+ log: log,
data: data{
SchemaVersion: CurrentSchemaVersion,
},
}, nil
}
-func openMeta(path string) (*Meta, error) {
+func openMeta(path string, log *log.Logger) (*Meta, error) {
exists, err := file.Exists(path)
if err != nil {
return nil, errors.WithMessage(err, "could not check for existence of index metadata")
}
if !exists {
- return createMeta(path)
+ return createMeta(path, log)
}
j, err := os.ReadFile(path)
@@ 61,6 63,7 @@ func openMeta(path string) (*Meta, error) {
}
meta := Meta{
path: path,
+ log: log,
}
err = json.Unmarshal(j, &meta.data)
if err != nil {
@@ 74,7 77,7 @@ func openMeta(path string) (*Meta, error) {
func (i *Meta) checkSchemaVersion() {
if i.SchemaVersion < CurrentSchemaVersion {
- slog.Warn(
+ i.log.Warn(
"Index schema version out of date, suggest re-indexing",
"schema_version",
i.SchemaVersion,
@@ 90,7 93,7 @@ func (i *Meta) Save() error {
if err != nil {
return errors.WithMessage(err, "could not prepare index metadata for saving")
}
- slog.Debug("saving index metadata", "path", i.path)
+ i.log.Debug("saving index metadata", "path", i.path)
err = os.WriteFile(i.path, j, 0o600)
if err != nil {
return errors.WithMessage(err, "could not save index metadata")
M internal/index/indexer.go => internal/index/indexer.go +16 -12
@@ 5,8 5,6 @@ import (
"context"
"encoding/gob"
"io/fs"
- "log"
- "log/slog"
"math"
"os"
"path"
@@ 14,6 12,8 @@ import (
"go.alanpearce.eu/searchix/internal/file"
"go.alanpearce.eu/searchix/internal/nix"
+ "go.alanpearce.eu/x/log"
+ "go.uber.org/zap"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ 30,6 30,7 @@ import (
type WriteIndex struct {
index bleve.Index
+ log *log.Logger
Meta *Meta
}
@@ 190,6 191,7 @@ func deleteIndex(dataRoot string) error {
type Options struct {
LowMemory bool
+ Logger *log.Logger
}
func OpenOrCreate(
@@ 198,7 200,7 @@ func OpenOrCreate(
options *Options,
) (*ReadIndex, *WriteIndex, bool, error) {
var err error
- bleve.SetLog(log.Default())
+ bleve.SetLog(zap.NewStdLog(options.Logger.Named("bleve").GetLogger()))
indexPath := path.Join(dataRoot, indexBaseName)
metaPath := path.Join(dataRoot, metaBaseName)
@@ 226,7 228,7 @@ func OpenOrCreate(
return nil, nil, false, err
}
- meta, err = createMeta(metaPath)
+ meta, err = createMeta(metaPath, options.Logger)
if err != nil {
return nil, nil, false, err
}
@@ 237,7 239,7 @@ func OpenOrCreate(
return nil, nil, exists, errors.WithMessagef(err, "could not open index at path %s", indexPath)
}
- meta, err = openMeta(metaPath)
+ meta, err = openMeta(metaPath, options.Logger)
if err != nil {
return nil, nil, exists, err
}
@@ 248,12 250,14 @@ func OpenOrCreate(
}
return &ReadIndex{
- idx,
- meta,
+ index: idx,
+ log: options.Logger,
+ meta: meta,
},
&WriteIndex{
- idx,
- meta,
+ index: idx,
+ log: options.Logger,
+ Meta: meta,
},
exists,
nil
@@ 280,7 284,7 @@ func (i *WriteIndex) Import(
for obj := range objects {
select {
case <-ctx.Done():
- slog.Warn("import aborted")
+ i.log.Warn("import aborted")
break outer
default:
@@ 305,7 309,7 @@ func (i *WriteIndex) Import(
field := document.NewTextFieldWithIndexingOptions("_data", nil, data.Bytes(), indexAPI.StoreField)
newDoc := doc.AddField(field)
- // slog.Debug("adding object to index", "name", opt.Name)
+ // log.Debug("adding object to index", "name", opt.Name)
err = batch.IndexAdvanced(newDoc)
if err != nil {
@@ 340,7 344,7 @@ func (i *WriteIndex) Flush(batch *bleve.Batch) error {
error: errors.New("no documents to flush"),
}
}
- slog.Debug("flushing batch", "size", size)
+ i.log.Debug("flushing batch", "size", size)
err := i.index.Batch(batch)
if err != nil {
M internal/index/search.go => internal/index/search.go +2 -0
@@ 7,6 7,7 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/nix"
+ "go.alanpearce.eu/x/log"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
@@ 29,6 30,7 @@ type Result struct {
type ReadIndex struct {
index bleve.Index
+ log *log.Logger
meta *Meta
}
M internal/server/dev.go => internal/server/dev.go +19 -15
@@ 3,37 3,41 @@ package server
import (
"fmt"
"io/fs"
- "log/slog"
"os"
"path/filepath"
"time"
"github.com/fsnotify/fsnotify"
"github.com/pkg/errors"
+ "go.alanpearce.eu/x/log"
)
type FileWatcher struct {
- *fsnotify.Watcher
+ watcher *fsnotify.Watcher
+ log *log.Logger
}
-func NewFileWatcher() (*FileWatcher, error) {
+func NewFileWatcher(log *log.Logger) (*FileWatcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, errors.WithMessage(err, "could not create watcher")
}
- return &FileWatcher{watcher}, nil
+ return &FileWatcher{
+ watcher,
+ log,
+ }, nil
}
-func (watcher FileWatcher) AddRecursive(from string) error {
- slog.Debug(fmt.Sprintf("watching files under %s", from))
+func (i FileWatcher) AddRecursive(from string) error {
+ i.log.Debug(fmt.Sprintf("watching files under %s", from))
err := filepath.WalkDir(from, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return errors.WithMessagef(err, "could not walk directory %s", path)
}
if entry.IsDir() {
- slog.Debug(fmt.Sprintf("adding directory %s to watcher", path))
- if err = watcher.Add(path); err != nil {
+ i.log.Debug(fmt.Sprintf("adding directory %s to watcher", path))
+ if err = i.watcher.Add(path); err != nil {
return errors.WithMessagef(err, "could not add directory %s to watcher", path)
}
}
@@ 44,18 48,18 @@ func (watcher FileWatcher) AddRecursive(from string) error {
return errors.WithMessage(err, "error walking directory tree")
}
-func (watcher FileWatcher) Start(callback func(string)) {
+func (i FileWatcher) Start(callback func(string)) {
for {
select {
- case event := <-watcher.Events:
+ case event := <-i.watcher.Events:
if event.Has(fsnotify.Create) || event.Has(fsnotify.Rename) {
f, err := os.Stat(event.Name)
if err != nil {
- slog.Error(fmt.Sprintf("error handling %s event: %v", event.Op.String(), err))
+ i.log.Error(fmt.Sprintf("error handling %s event: %v", event.Op.String(), err))
} else if f.IsDir() {
- err = watcher.Add(event.Name)
+ err = i.watcher.Add(event.Name)
if err != nil {
- slog.Error(fmt.Sprintf("error adding new folder to watcher: %v", err))
+ i.log.Error(fmt.Sprintf("error adding new folder to watcher: %v", err))
}
}
}
@@ 63,8 67,8 @@ func (watcher FileWatcher) Start(callback func(string)) {
callback(event.Name)
time.Sleep(500 * time.Millisecond)
}
- case err := <-watcher.Errors:
- slog.Error(fmt.Sprintf("error in watcher: %v", err))
+ case err := <-i.watcher.Errors:
+ i.log.Error(fmt.Sprintf("error in watcher: %v", err))
}
}
}
M internal/server/error.go => internal/server/error.go +3 -2
@@ 1,15 1,16 @@
package server
import (
- "log/slog"
"net/http"
"go.alanpearce.eu/searchix/internal/components"
"go.alanpearce.eu/searchix/internal/config"
+ "go.alanpearce.eu/x/log"
)
func createErrorHandler(
config *config.Config,
+ log *log.Logger,
) func(http.ResponseWriter, *http.Request, string, int) {
return func(w http.ResponseWriter, r *http.Request, message string, code int) {
var err error
@@ 31,7 32,7 @@ func createErrorHandler(
err = components.ErrorPage(indexData).Render(r.Context(), w)
}
if err != nil {
- slog.Error(
+ log.Error(
"error rendering error page template",
"error",
err,
M internal/server/logging.go => internal/server/logging.go +9 -6
@@ 1,11 1,10 @@
package server
import (
- "fmt"
- "io"
"net/http"
"github.com/pkg/errors"
+ "go.alanpearce.eu/x/log"
)
type LoggingResponseWriter struct {
@@ 42,7 41,7 @@ func NewLoggingResponseWriter(w http.ResponseWriter) *LoggingResponseWriter {
type wrappedHandlerOptions struct {
defaultHostname string
- logger io.Writer
+ logger *log.Logger
}
func wrapHandlerWithLogging(wrappedHandler http.Handler, opts wrappedHandlerOptions) http.Handler {
@@ 54,13 53,17 @@ func wrapHandlerWithLogging(wrappedHandler http.Handler, opts wrappedHandlerOpti
lw := NewLoggingResponseWriter(w)
wrappedHandler.ServeHTTP(lw, r)
statusCode := lw.statusCode
- fmt.Fprintf(
- opts.logger,
- "%s %s %d %s %s\n",
+ opts.logger.Info(
+ "http request",
+ "scheme",
scheme,
+ "method",
r.Method,
+ "code",
statusCode,
+ "path",
r.URL.Path,
+ "location",
lw.Header().Get("Location"),
)
})
M internal/server/mux.go => internal/server/mux.go +10 -18
@@ 4,12 4,9 @@ import (
"context"
"encoding/xml"
"fmt"
- "io"
- "log/slog"
"math"
"net/http"
"net/url"
- "os"
"path"
"strconv"
"time"
@@ 19,11 16,11 @@ import (
"go.alanpearce.eu/searchix/internal/config"
search "go.alanpearce.eu/searchix/internal/index"
"go.alanpearce.eu/searchix/internal/opensearch"
+ "go.alanpearce.eu/x/log"
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/osdevisnot/sorvor/pkg/livereload"
"github.com/pkg/errors"
- "github.com/shengyanli1982/law"
)
type HTTPError struct {
@@ 57,6 54,7 @@ func sortSources(ss map[string]*config.Source) {
func NewMux(
cfg *config.Config,
index *search.ReadIndex,
+ log *log.Logger,
liveReload bool,
) (*http.ServeMux, error) {
if cfg == nil {
@@ 70,7 68,7 @@ func NewMux(
})
sortSources(cfg.Importer.Sources)
- errorHandler := createErrorHandler(cfg)
+ errorHandler := createErrorHandler(cfg, log)
top := http.NewServeMux()
mux := http.NewServeMux()
@@ 118,7 116,7 @@ func NewMux(
return
}
- slog.Error("search error", "error", err)
+ log.Error("search error", "error", err)
errorHandler(w, r, err.Error(), http.StatusInternalServerError)
return
@@ 177,7 175,7 @@ func NewMux(
err = components.ResultsPage(tdata).Render(r.Context(), w)
}
if err != nil {
- slog.Error("template error", "template", importerType, "error", err)
+ log.Error("template error", "template", importerType, "error", err)
errorHandler(w, r, err.Error(), http.StatusInternalServerError)
}
} else {
@@ 258,7 256,7 @@ func NewMux(
err = components.DetailPage(tdata.TemplateData, *doc).Render(r.Context(), w)
}
if err != nil {
- slog.Error("template error", "template", importerSingular, "error", err)
+ log.Error("template error", "template", importerSingular, "error", err)
errorHandler(w, r, err.Error(), http.StatusInternalServerError)
}
}
@@ 332,7 330,7 @@ func NewMux(
liveReload := livereload.New()
liveReload.Start()
top.Handle("/livereload", liveReload)
- fw, err := NewFileWatcher()
+ fw, err := NewFileWatcher(log.Named("watcher"))
if err != nil {
return nil, errors.WithMessage(err, "could not create file watcher")
}
@@ 341,29 339,23 @@ func NewMux(
return nil, errors.WithMessage(err, "could not add directory to file watcher")
}
go fw.Start(func(filename string) {
- slog.Debug(fmt.Sprintf("got filename %s", filename))
+ log.Debug(fmt.Sprintf("got filename %s", filename))
if match, _ := path.Match("frontend/static/*", filename); match {
err := frontend.Rehash()
if err != nil {
- slog.Error("failed to re-hash frontend assets", "error", err)
+ log.Error("failed to re-hash frontend assets", "error", err)
}
}
liveReload.Reload()
})
}
- var logWriter io.Writer
- if cfg.Web.Environment == "production" {
- logWriter = law.NewWriteAsyncer(os.Stdout, nil)
- } else {
- logWriter = os.Stdout
- }
top.Handle("/",
AddHeadersMiddleware(
sentryHandler.Handle(
wrapHandlerWithLogging(mux, wrappedHandlerOptions{
defaultHostname: cfg.Web.BaseURL.Hostname(),
- logger: logWriter,
+ logger: log,
}),
),
cfg,
M internal/server/server.go => internal/server/server.go +15 -8
@@ 2,7 2,6 @@ package server
import (
"context"
- "log/slog"
"net"
"net/http"
"strconv"
@@ 10,6 9,7 @@ import (
"go.alanpearce.eu/searchix/internal/config"
"go.alanpearce.eu/searchix/internal/index"
+ "go.alanpearce.eu/x/log"
"github.com/pkg/errors"
"golang.org/x/net/http2"
@@ 18,18 18,25 @@ import (
type Server struct {
cfg *config.Config
+ log *log.Logger
server *http.Server
listener net.Listener
}
-func New(conf *config.Config, index *index.ReadIndex, liveReload bool) (*Server, error) {
- mux, err := NewMux(conf, index, liveReload)
+func New(
+ conf *config.Config,
+ index *index.ReadIndex,
+ log *log.Logger,
+ liveReload bool,
+) (*Server, error) {
+ mux, err := NewMux(conf, index, log, liveReload)
if err != nil {
return nil, err
}
return &Server{
cfg: conf,
+ log: log,
server: &http.Server{
Handler: http.MaxBytesHandler(
h2c.NewHandler(mux, &http2.Server{
@@ 56,7 63,7 @@ func (s *Server) Start() error {
s.listener = l
if s.cfg.Web.Environment == "development" {
- slog.Info(
+ s.log.Info(
"server listening on",
"url",
s.cfg.Web.BaseURL.String(),
@@ 71,19 78,19 @@ func (s *Server) Start() error {
}
func (s *Server) Stop() chan struct{} {
- slog.Debug("stop called")
+ s.log.Debug("stop called")
idleConnsClosed := make(chan struct{})
go func() {
- slog.Debug("shutting down server")
+ s.log.Debug("shutting down server")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
err := s.server.Shutdown(ctx)
- slog.Debug("server shut down")
+ s.log.Debug("server shut down")
if err != nil {
// Error from closing listeners, or context timeout:
- slog.Error("error shutting down server", "error", err)
+ s.log.Error("error shutting down server", "error", err)
}
s.listener.Close()
close(idleConnsClosed)
M justfile => justfile +2 -2
@@ 47,7 47,7 @@ dev:
modd
reindex:
- wgo run --exit ./cmd/searchix-web --config config.toml --replace
+ wgo run --exit ./cmd/searchix-web --config config.toml --replace --dev
update:
- wgo run --exit ./cmd/searchix-web --config config.toml --update
+ wgo run --exit ./cmd/searchix-web --config config.toml --update --dev
M modd.conf => modd.conf +1 -1
@@ 1,4 1,4 @@
**/*.go !**/*_templ.go config.toml {
daemon +sigint: templ generate --watch --proxy="http://localhost:3000" --open-browser=false \
- --cmd="go run ./cmd/searchix-web --live --config config.toml"
+ --cmd="go run ./cmd/searchix-web --dev --config config.toml"
}
M searchix.go => searchix.go +25 -23
@@ 2,8 2,6 @@ package searchix
import (
"context"
- "log"
- "log/slog"
"slices"
"sync"
"time"
@@ 12,6 10,7 @@ import (
"go.alanpearce.eu/searchix/internal/importer"
"go.alanpearce.eu/searchix/internal/index"
"go.alanpearce.eu/searchix/internal/server"
+ "go.alanpearce.eu/x/log"
"github.com/getsentry/sentry-go"
"github.com/pelletier/go-toml/v2"
@@ 42,6 41,7 @@ type IndexOptions struct {
Update bool
Replace bool
LowMemory bool
+ Logger *log.Logger
}
func (s *Server) SetupIndex(options *IndexOptions) error {
@@ 58,6 58,7 @@ func (s *Server) SetupIndex(options *IndexOptions) error {
options.Replace,
&index.Options{
LowMemory: options.LowMemory,
+ Logger: options.Logger.Named("index"),
},
)
if err != nil {
@@ 67,7 68,7 @@ func (s *Server) SetupIndex(options *IndexOptions) error {
s.writeIndex = write
if !exists || options.Replace || options.Update {
- slog.Info(
+ s.log.Info(
"Starting build job",
"new",
!exists,
@@ 76,7 77,13 @@ func (s *Server) SetupIndex(options *IndexOptions) error {
"update",
options.Update,
)
- err = importer.Start(s.cfg, write, options.Replace || options.Update, nil)
+ err = importer.Start(
+ s.cfg,
+ s.log.Named("importer"),
+ write,
+ options.Replace || options.Update,
+ nil,
+ )
if err != nil {
return errors.Wrap(err, "Failed to build index")
}
@@ 97,14 104,14 @@ func (s *Server) SetupIndex(options *IndexOptions) error {
return slices.Contains(cfgEnabledSources, s)
})
if len(newSources) > 0 {
- slog.Info("adding new sources", "sources", newSources)
- err := importer.Start(s.cfg, write, false, &newSources)
+ s.log.Info("adding new sources", "sources", newSources)
+ err := importer.Start(s.cfg, options.Logger.Named("importer"), write, false, &newSources)
if err != nil {
return errors.Wrap(err, "Failed to update index with new sources")
}
}
if len(retiredSources) > 0 {
- slog.Info("removing retired sources", "sources", retiredSources)
+ s.log.Info("removing retired sources", "sources", retiredSources)
for _, s := range retiredSources {
err := write.DeleteBySource(s)
if err != nil {
@@ 122,19 129,13 @@ type Server struct {
sv *server.Server
wg *sync.WaitGroup
cfg *config.Config
+ log *log.Logger
sentryHub *sentry.Hub
readIndex *index.ReadIndex
writeIndex *index.WriteIndex
}
-func New(cfg *config.Config) (*Server, error) {
- slog.SetLogLoggerLevel(cfg.LogLevel)
- if cfg.Web.Environment == "production" {
- log.SetFlags(0)
- } else {
- log.SetFlags(log.LstdFlags)
- }
-
+func New(cfg *config.Config, log *log.Logger) (*Server, error) {
err := sentry.Init(sentry.ClientOptions{
EnableTracing: true,
TracesSampleRate: 0.01,
@@ 142,11 143,12 @@ func New(cfg *config.Config) (*Server, error) {
Environment: cfg.Web.Environment,
})
if err != nil {
- slog.Warn("could not initialise sentry", "error", err)
+ log.Warn("could not initialise sentry", "error", err)
}
return &Server{
cfg: cfg,
+ log: log,
sentryHub: sentry.CurrentHub(),
}, nil
}
@@ 170,27 172,27 @@ func (s *Server) startUpdateTimer(
s.wg.Add(1)
nextRun := nextOccurrenceOfLocalTime(s.cfg.Importer.UpdateAt.LocalTime)
for {
- slog.Debug("scheduling next run", "next-run", nextRun)
+ s.log.Debug("scheduling next run", "next-run", nextRun)
select {
case <-ctx.Done():
- slog.Debug("stopping scheduler")
+ s.log.Debug("stopping scheduler")
s.wg.Done()
return
case <-time.After(time.Until(nextRun)):
}
s.wg.Add(1)
- slog.Info("updating index")
+ s.log.Info("updating index")
eventID := localHub.CaptureCheckIn(&sentry.CheckIn{
MonitorSlug: monitorSlug,
Status: sentry.CheckInStatusInProgress,
}, monitorConfig)
- err = importer.Start(s.cfg, s.writeIndex, false, nil)
+ err = importer.Start(s.cfg, s.log.Named("importer"), s.writeIndex, false, nil)
s.wg.Done()
if err != nil {
- slog.Warn("error updating index", "error", err)
+ s.log.Warn("error updating index", "error", err)
localHub.CaptureException(err)
localHub.CaptureCheckIn(&sentry.CheckIn{
@@ 199,7 201,7 @@ func (s *Server) startUpdateTimer(
Status: sentry.CheckInStatusError,
}, monitorConfig)
} else {
- slog.Info("update complete")
+ s.log.Info("update complete")
localHub.CaptureCheckIn(&sentry.CheckIn{
ID: *eventID,
@@ 214,7 216,7 @@ func (s *Server) startUpdateTimer(
func (s *Server) Start(ctx context.Context, liveReload bool) error {
var err error
- s.sv, err = server.New(s.cfg, s.readIndex, liveReload)
+ s.sv, err = server.New(s.cfg, s.readIndex, s.log.Named("server"), liveReload)
if err != nil {
return errors.Wrap(err, "error setting up server")
}