diff options
author | Jakob Borg <jakob@kastelo.net> | 2024-05-18 19:31:49 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-18 20:31:49 +0300 |
commit | ba6ac2f604eb1cd27764460b687537c5e40aaaf8 (patch) | |
tree | 99fbd81b711e0c8819a5bd64376bd0197b025054 | |
parent | 57d399317e050189d76068872cf791f1cd1c0e24 (diff) |
lib/geoip, cmd/relaypoolsrv, cmd/ursrv: Automatically manage GeoIP updates (#9342)
This adds a small package `geoip` which knows how to download and manage
the Maxmind GeoLite2 database we use. This removes the need for various
scripts to download and manage the geoip database, something that today
happens on Docker startup for the relay pool server and using various
hand written hacks for the usage reporting server.
The database is downloaded when needed and then refreshed on a
best-effort basis weekly.
-rw-r--r-- | Dockerfile.strelaypoolsrv | 10 | ||||
-rw-r--r-- | cmd/strelaypoolsrv/main.go | 40 | ||||
-rw-r--r-- | cmd/ursrv/serve/serve.go | 43 | ||||
-rw-r--r-- | go.mod | 4 | ||||
-rw-r--r-- | go.sum | 7 | ||||
-rw-r--r-- | lib/geoip/geoip.go | 124 | ||||
-rw-r--r-- | lib/geoip/geoip_test.go | 36 | ||||
-rwxr-xr-x | script/strelaypoolsrv-entrypoint.sh | 10 |
8 files changed, 212 insertions, 62 deletions
diff --git a/Dockerfile.strelaypoolsrv b/Dockerfile.strelaypoolsrv index f7e2760e77..a0ad1fd6dd 100644 --- a/Dockerfile.strelaypoolsrv +++ b/Dockerfile.strelaypoolsrv @@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \ EXPOSE 8080 -RUN apk add --no-cache ca-certificates su-exec curl -ENV PUID=1000 PGID=1000 MAXMIND_KEY= - -RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv -USER 1000 - COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv -COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh -WORKDIR /var/strelaypoolsrv -ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"] +ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"] diff --git a/cmd/strelaypoolsrv/main.go b/cmd/strelaypoolsrv/main.go index 6d574e3e88..6f1e559ef8 100644 --- a/cmd/strelaypoolsrv/main.go +++ b/cmd/strelaypoolsrv/main.go @@ -21,12 +21,12 @@ import ( "time" lru "github.com/hashicorp/golang-lru/v2" - "github.com/oschwald/geoip2-golang" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto" "github.com/syncthing/syncthing/lib/assets" _ "github.com/syncthing/syncthing/lib/automaxprocs" + "github.com/syncthing/syncthing/lib/geoip" "github.com/syncthing/syncthing/lib/httpcache" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" @@ -100,11 +100,12 @@ var ( debug bool permRelaysFile string ipHeader string - geoipPath string proto string statsRefresh = time.Minute requestQueueLen = 64 requestProcessors = 8 + geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY") + geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID")) requests chan request @@ -130,34 +131,38 @@ func main() { flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays") flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays") flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.") - flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database") flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6") flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats") flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests") flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines") + flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database") flag.Parse() requests = make(chan request, requestQueueLen) + geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir()) + if err != nil { + log.Fatalln("Failed to create GeoIP provider:", err) + } + go geoip.Serve(context.TODO()) var listener net.Listener - var err error if permRelaysFile != "" { - permanentRelays = loadRelays(permRelaysFile) + permanentRelays = loadRelays(permRelaysFile, geoip) } testCert = createTestCertificate() for i := 0; i < requestProcessors; i++ { - go requestProcessor() + go requestProcessor(geoip) } // Load relays from cache in the background. // Load them in a serial fashion to make sure any genuine requests // are not dropped. go func() { - for _, relay := range loadRelays(knownRelaysFile) { + for _, relay := range loadRelays(knownRelaysFile, geoip) { resultChan := make(chan result) requests <- request{relay, resultChan, nil} result := <-resultChan @@ -425,19 +430,19 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) { } } -func requestProcessor() { +func requestProcessor(geoip *geoip.Provider) { for request := range requests { if request.queueTimer != nil { request.queueTimer.ObserveDuration() } timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test")) - handleRelayTest(request) + handleRelayTest(request, geoip) timer.ObserveDuration() } } -func handleRelayTest(request request) { +func handleRelayTest(request request, geoip *geoip.Provider) { if debug { log.Println("Request for", request.relay) } @@ -450,7 +455,7 @@ func handleRelayTest(request request) { } stats := fetchStats(request.relay) - location := getLocation(request.relay.uri.Host) + location := getLocation(request.relay.uri.Host, geoip) mut.Lock() if stats != nil { @@ -523,7 +528,7 @@ func evict(relay *relay) func() { } } -func loadRelays(file string) []*relay { +func loadRelays(file string, geoip *geoip.Provider) []*relay { content, err := os.ReadFile(file) if err != nil { log.Println("Failed to load relays: " + err.Error()) @@ -547,7 +552,7 @@ func loadRelays(file string) []*relay { relays = append(relays, &relay{ URL: line, - Location: getLocation(uri.Host), + Location: getLocation(uri.Host, geoip), uri: uri, }) if debug { @@ -580,21 +585,16 @@ func createTestCertificate() tls.Certificate { return cert } -func getLocation(host string) location { +func getLocation(host string, geoip *geoip.Provider) location { timer := prometheus.NewTimer(locationLookupSeconds) defer timer.ObserveDuration() - db, err := geoip2.Open(geoipPath) - if err != nil { - return location{} - } - defer db.Close() addr, err := net.ResolveTCPAddr("tcp", host) if err != nil { return location{} } - city, err := db.City(addr.IP) + city, err := geoip.City(addr.IP) if err != nil { return location{} } diff --git a/cmd/ursrv/serve/serve.go b/cmd/ursrv/serve/serve.go index b7d51db3b5..14d3123b12 100644 --- a/cmd/ursrv/serve/serve.go +++ b/cmd/ursrv/serve/serve.go @@ -8,6 +8,7 @@ package serve import ( "bytes" + "context" "database/sql" "embed" "encoding/json" @@ -17,6 +18,7 @@ import ( "log" "net" "net/http" + "os" "regexp" "sort" "strconv" @@ -26,20 +28,21 @@ import ( "unicode" _ "github.com/lib/pq" // PostgreSQL driver - "github.com/oschwald/geoip2-golang" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/text/cases" "golang.org/x/text/language" + "github.com/syncthing/syncthing/lib/geoip" "github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/ur/contract" ) type CLI struct { - Debug bool `env:"UR_DEBUG"` - DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` - Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` - GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"` + Debug bool `env:"UR_DEBUG"` + DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` + Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` + GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"` + GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"` } //go:embed static @@ -189,10 +192,16 @@ func (cli *CLI) Run() error { log.Fatalln("listen:", err) } + geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir()) + if err != nil { + log.Fatalln("geoip:", err) + } + go geoip.Serve(context.TODO()) + srv := &server{ - db: db, - debug: cli.Debug, - geoIPPath: cli.GeoIPPath, + db: db, + debug: cli.Debug, + geoip: geoip, } http.HandleFunc("/", srv.rootHandler) http.HandleFunc("/newdata", srv.newDataHandler) @@ -213,9 +222,9 @@ func (cli *CLI) Run() error { } type server struct { - debug bool - db *sql.DB - geoIPPath string + debug bool + db *sql.DB + geoip *geoip.Provider cacheMut sync.Mutex cachedIndex []byte @@ -238,7 +247,7 @@ func (s *server) cacheRefresher() { } func (s *server) refreshCacheLocked() error { - rep := getReport(s.db, s.geoIPPath) + rep := getReport(s.db, s.geoip) buf := new(bytes.Buffer) err := tpl.Execute(buf, rep) if err != nil { @@ -492,15 +501,7 @@ type weightedLocation struct { Weight int `json:"weight"` } -func getReport(db *sql.DB, geoIPPath string) map[string]interface{} { - geoip, err := geoip2.Open(geoIPPath) - if err != nil { - log.Println("opening geoip db", err) - geoip = nil - } else { - defer geoip.Close() - } - +func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} { nodes := 0 countriesTotal := 0 var versions []string @@ -24,6 +24,7 @@ require ( github.com/lib/pq v1.10.9 github.com/maruel/panicparse/v2 v2.3.1 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 + github.com/maxmind/geoipupdate/v6 v6.1.0 github.com/minio/sha256-simd v1.0.1 github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 github.com/oschwald/geoip2-golang v1.9.0 @@ -52,6 +53,7 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect @@ -60,13 +62,13 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/nxadm/tail v1.4.11 // indirect github.com/onsi/ginkgo/v2 v2.17.1 // indirect github.com/onsi/gomega v1.31.1 // indirect @@ -22,6 +22,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE= github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg= github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M= github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -34,7 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,6 +63,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -131,6 +134,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= +github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA= +github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= diff --git a/lib/geoip/geoip.go b/lib/geoip/geoip.go new file mode 100644 index 0000000000..3400587e0b --- /dev/null +++ b/lib/geoip/geoip.go @@ -0,0 +1,124 @@ +// Copyright (C) 2024 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Package geoip provides an automatically updating MaxMind GeoIP2 database +// provider. +package geoip + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sync" + "time" + + "github.com/maxmind/geoipupdate/v6/pkg/geoipupdate" + "github.com/oschwald/geoip2-golang" +) + +type Provider struct { + edition string + accountID int + licenseKey string + refreshInterval time.Duration + directory string + + mut sync.Mutex + currentDBDir string + db *geoip2.Reader +} + +// NewGeoLite2CityProvider returns a new GeoIP2 database provider for the +// GeoLite2-City database. The database will be stored in the given +// directory (which should exist) and refreshed every 7 days. +func NewGeoLite2CityProvider(ctx context.Context, accountID int, licenseKey string, directory string) (*Provider, error) { + p := &Provider{ + edition: "GeoLite2-City", + accountID: accountID, + licenseKey: licenseKey, + refreshInterval: 7 * 24 * time.Hour, + directory: directory, + } + + if err := p.download(ctx); err != nil { + return nil, err + } + + return p, nil +} + +func (p *Provider) City(ip net.IP) (*geoip2.City, error) { + p.mut.Lock() + defer p.mut.Unlock() + + if p.db == nil { + return nil, errors.New("database not open") + } + + return p.db.City(ip) +} + +// Serve downloads the GeoIP2 database and keeps it up to date. It will return +// when the context is canceled. +func (p *Provider) Serve(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + + case <-time.After(p.refreshInterval): + if err := p.download(ctx); err != nil { + return err + } + } + } +} + +func (p *Provider) download(ctx context.Context) error { + newSubdir, err := os.MkdirTemp(p.directory, "geoipupdate") + if err != nil { + return fmt.Errorf("download: %w", err) + } + + cfg := &geoipupdate.Config{ + URL: "https://updates.maxmind.com", + DatabaseDirectory: newSubdir, + LockFile: filepath.Join(newSubdir, "geoipupdate.lock"), + RetryFor: 5 * time.Minute, + Parallelism: 1, + AccountID: p.accountID, + LicenseKey: p.licenseKey, + EditionIDs: []string{p.edition}, + } + + if err := geoipupdate.NewClient(cfg).Run(ctx); err != nil { + return fmt.Errorf("download: %w", err) + } + + dbPath := filepath.Join(newSubdir, p.edition+".mmdb") + db, err := geoip2.Open(dbPath) + if err != nil { + return fmt.Errorf("open downloaded db: %w", err) + } + + p.mut.Lock() + prevDBDir := p.currentDBDir + if p.db != nil { + p.db.Close() + } + p.currentDBDir = newSubdir + p.db = db + p.mut.Unlock() + + if prevDBDir != "" { + _ = os.RemoveAll(p.currentDBDir) + } + + return nil +} diff --git a/lib/geoip/geoip_test.go b/lib/geoip/geoip_test.go new file mode 100644 index 0000000000..bf22c1c363 --- /dev/null +++ b/lib/geoip/geoip_test.go @@ -0,0 +1,36 @@ +// Copyright (C) 2024 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package geoip + +import ( + "context" + "net" + "os" + "strconv" + "testing" +) + +func TestDownloadAndOpen(t *testing.T) { + acctID, _ := strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID")) + if acctID == 0 { + t.Skip("No account ID set") + } + license := os.Getenv("GEOIP_LICENSE_KEY") + if license == "" { + t.Skip("No license key set") + } + + p, err := NewGeoLite2CityProvider(context.Background(), acctID, license, t.TempDir()) + if err != nil { + t.Fatal(err) + } + + _, err = p.City(net.ParseIP("8.8.8.8")) + if err != nil { + t.Fatal(err) + } +} diff --git a/script/strelaypoolsrv-entrypoint.sh b/script/strelaypoolsrv-entrypoint.sh deleted file mode 100755 index 945c77d5b0..0000000000 --- a/script/strelaypoolsrv-entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -eu - -if [ "$MAXMIND_KEY" != "" ] ; then - curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_KEY}&suffix=tar.gz" \ - | tar --strip-components 1 -zxv -fi - -exec "$@" |