diff options
58 files changed, 2453 insertions, 1461 deletions
diff --git a/.github/workflows/build-infra-dockers.yaml b/.github/workflows/build-infra-dockers.yaml index d785153a4..f899a3524 100644 --- a/.github/workflows/build-infra-dockers.yaml +++ b/.github/workflows/build-infra-dockers.yaml @@ -7,7 +7,7 @@ on: - infra-* env: - GO_VERSION: "~1.22.0" + GO_VERSION: "~1.22.3" CGO_ENABLED: "0" BUILD_USER: docker BUILD_HOST: github.syncthing.net diff --git a/.github/workflows/build-syncthing.yaml b/.github/workflows/build-syncthing.yaml index eba9709d7..984d222b7 100644 --- a/.github/workflows/build-syncthing.yaml +++ b/.github/workflows/build-syncthing.yaml @@ -12,7 +12,7 @@ env: # The go version to use for builds. We set check-latest to true when # installing, so we get the latest patch version that matches the # expression. - GO_VERSION: "~1.22.0" + GO_VERSION: "~1.22.3" # Optimize compatibility on the slow archictures. GO386: softfloat @@ -48,7 +48,7 @@ jobs: runner: ["windows-latest", "ubuntu-latest", "macos-latest"] # The oldest version in this list should match what we have in our go.mod. # Variables don't seem to be supported here, or we could have done something nice. - go: ["~1.21.7", "~1.22.0"] + go: ["~1.21.7", "~1.22.3"] runs-on: ${{ matrix.runner }} steps: - name: Set git to use LF @@ -642,19 +642,20 @@ jobs: cd packages "$GITHUB_WORKSPACE/tools/generate-release-json" "$BASE_URL" > nightly.json env: - BASE_URL: https://syncthing.ams3.digitaloceanspaces.com/nightly/ + BASE_URL: ${{ secrets.NIGHTLY_BASE_URL }} - name: Push artifacts uses: docker://docker.io/rclone/rclone:latest env: - RCLONE_CONFIG_SPACES_TYPE: s3 - RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean - RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }} - RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }} - RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com - RCLONE_CONFIG_SPACES_ACL: public-read + RCLONE_CONFIG_OBJSTORE_TYPE: s3 + RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }} + RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }} + RCLONE_CONFIG_OBJSTORE_ACL: public-read with: - args: sync packages spaces:syncthing/nightly + args: sync packages objstore:${{ secrets.S3_BUCKET }}/nightly # # Push release artifacts to Spaces @@ -696,29 +697,31 @@ jobs: version=$(go run build.go version) echo "VERSION=$version" >> $GITHUB_ENV - - name: Push to Spaces (${{ env.VERSION }}) + - name: Push to object store (${{ env.VERSION }}) uses: docker://docker.io/rclone/rclone:latest env: - RCLONE_CONFIG_SPACES_TYPE: s3 - RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean - RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }} - RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }} - RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com - RCLONE_CONFIG_SPACES_ACL: public-read + RCLONE_CONFIG_OBJSTORE_TYPE: s3 + RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }} + RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }} + RCLONE_CONFIG_OBJSTORE_ACL: public-read with: - args: sync packages spaces:syncthing/release/${{ env.VERSION }} + args: sync packages objstore:${{ secrets.S3_BUCKET }}/release/${{ env.VERSION }} - - name: Push to Spaces (latest) + - name: Push to object store (latest) uses: docker://docker.io/rclone/rclone:latest env: - RCLONE_CONFIG_SPACES_TYPE: s3 - RCLONE_CONFIG_SPACES_PROVIDER: DigitalOcean - RCLONE_CONFIG_SPACES_ACCESS_KEY_ID: ${{ secrets.SPACES_KEY }} - RCLONE_CONFIG_SPACES_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET }} - RCLONE_CONFIG_SPACES_ENDPOINT: ams3.digitaloceanspaces.com - RCLONE_CONFIG_SPACES_ACL: public-read - with: - args: sync spaces:syncthing/release/${{ env.VERSION }} spaces:syncthing/release/latest + RCLONE_CONFIG_OBJSTORE_TYPE: s3 + RCLONE_CONFIG_OBJSTORE_PROVIDER: ${{ secrets.S3_PROVIDER }} + RCLONE_CONFIG_OBJSTORE_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + RCLONE_CONFIG_OBJSTORE_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + RCLONE_CONFIG_OBJSTORE_ENDPOINT: ${{ secrets.S3_ENDPOINT }} + RCLONE_CONFIG_OBJSTORE_REGION: ${{ secrets.S3_REGION }} + RCLONE_CONFIG_OBJSTORE_ACL: public-read + with: + args: sync objstore:${{ secrets.S3_BUCKET }}/release/${{ env.VERSION }} objstore:${{ secrets.S3_BUCKET }}/release/latest # # Build and push to Docker Hub @@ -209,6 +209,7 @@ Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com> Lode Hoste (Zillode) <zillode@zillode.be> Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com> LSmithx2 <42276854+lsmithx2@users.noreply.github.com> +luchenhan <168071714+luchenhan@users.noreply.github.com> Lukas Lihotzki <lukas@lihotzki.de> Luke Hamburg <1992842+luckman212@users.noreply.github.com> luzpaz <luzpaz@users.noreply.github.com> @@ -336,6 +337,7 @@ Vil Brekin (Vilbrekin) <vilbrekin@gmail.com> villekalliomaki <53118179+villekalliomaki@users.noreply.github.com> Vladimir Rusinov <vrusinov@google.com> <vladimir.rusinov@gmail.com> wangguoliang <liangcszzu@163.com> +WangXi <xib1102@icloud.com> Will Rouesnel <wrouesnel@wrouesnel.com> William A. Kennington III (wkennington) <william@wkennington.com> wouter bolsterlee <wouter@bolsterl.ee> diff --git a/Dockerfile.strelaypoolsrv b/Dockerfile.strelaypoolsrv index f7e2760e7..a0ad1fd6d 100644 --- a/Dockerfile.strelaypoolsrv +++ b/Dockerfile.strelaypoolsrv @@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \ EXPOSE 8080 -RUN apk add --no-cache ca-certificates su-exec curl -ENV PUID=1000 PGID=1000 MAXMIND_KEY= - -RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv -USER 1000 - COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv -COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh -WORKDIR /var/strelaypoolsrv -ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"] +ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"] diff --git a/cmd/stcrashreceiver/main.go b/cmd/stcrashreceiver/main.go index 657d19e91..1a22313ab 100644 --- a/cmd/stcrashreceiver/main.go +++ b/cmd/stcrashreceiver/main.go @@ -21,11 +21,14 @@ import ( "net/http" "os" "path/filepath" + "regexp" + "strings" "github.com/alecthomas/kong" raven "github.com/getsentry/raven-go" "github.com/prometheus/client_golang/prometheus/promhttp" _ "github.com/syncthing/syncthing/lib/automaxprocs" + "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/sha256" "github.com/syncthing/syncthing/lib/ur" ) @@ -33,13 +36,15 @@ import ( const maxRequestSize = 1 << 20 // 1 MiB type cli struct { - Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."` - DSN string `help:"Sentry DSN" env:"SENTRY_DSN"` - Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"` - MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"` - MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"` - SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"` - DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"` + Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."` + DSN string `help:"Sentry DSN" env:"SENTRY_DSN"` + Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"` + MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"` + MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"` + SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"` + DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"` + MetricsListen string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"` + IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"` } func main() { @@ -62,19 +67,38 @@ func main() { } go ss.Serve(context.Background()) + var ip *ignorePatterns + if params.IngorePatterns != "" { + var err error + ip, err = loadIgnorePatterns(params.IngorePatterns) + if err != nil { + log.Fatalf("Failed to load ignore patterns: %v", err) + } + } + cr := &crashReceiver{ store: ds, sentry: ss, + ignore: ip, } mux.Handle("/", cr) mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("OK")) }) - mux.Handle("/metrics", promhttp.Handler()) + + if params.MetricsListen != "" { + mmux := http.NewServeMux() + mmux.Handle("/metrics", promhttp.Handler()) + go func() { + if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil { + log.Fatalln("HTTP serve metrics:", err) + } + }() + } if params.DSN != "" { - mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"))) + mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip)) } log.SetOutput(os.Stdout) @@ -83,7 +107,7 @@ func main() { } } -func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) { +func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) { result := "failure" defer func() { @@ -98,6 +122,11 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht return } + if ignore.match(bs) { + result = "ignored" + return + } + var reports []ur.FailureReport err = json.Unmarshal(bs, &reports) if err != nil { @@ -110,7 +139,7 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht return } - version, err := parseVersion(reports[0].Version) + version, err := build.ParseVersion(reports[0].Version) if err != nil { http.Error(w, err.Error(), 400) return @@ -158,3 +187,42 @@ func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, } return reportServer + path, nil } + +type ignorePatterns struct { + patterns []*regexp.Regexp +} + +func loadIgnorePatterns(path string) (*ignorePatterns, error) { + bs, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var patterns []*regexp.Regexp + for _, line := range strings.Split(string(bs), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + re, err := regexp.Compile(line) + if err != nil { + return nil, err + } + patterns = append(patterns, re) + } + + log.Printf("Loaded %d ignore patterns", len(patterns)) + return &ignorePatterns{patterns: patterns}, nil +} + +func (i *ignorePatterns) match(report []byte) bool { + if i == nil { + return false + } + for _, re := range i.patterns { + if re.Match(report) { + return true + } + } + return false +} diff --git a/cmd/stcrashreceiver/sentry.go b/cmd/stcrashreceiver/sentry.go index dadd9f333..f8bc22741 100644 --- a/cmd/stcrashreceiver/sentry.go +++ b/cmd/stcrashreceiver/sentry.go @@ -18,6 +18,7 @@ import ( raven "github.com/getsentry/raven-go" "github.com/maruel/panicparse/v2/stack" + "github.com/syncthing/syncthing/lib/build" ) const reportServer = "https://crash.syncthing.net/report/" @@ -105,7 +106,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { return nil, errors.New("no first line") } - version, err := parseVersion(string(parts[0])) + version, err := build.ParseVersion(string(parts[0])) if err != nil { return nil, err } @@ -143,12 +144,12 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { } // Lock the source code loader to the version we are processing here. - if version.commit != "" { + if version.Commit != "" { // We have a commit hash, so we know exactly which source to use - loader.LockWithVersion(version.commit) - } else if strings.HasPrefix(version.tag, "v") { + loader.LockWithVersion(version.Commit) + } else if strings.HasPrefix(version.Tag, "v") { // Lets hope the tag is close enough - loader.LockWithVersion(version.tag) + loader.LockWithVersion(version.Tag) } else { // Last resort loader.LockWithVersion("main") @@ -215,106 +216,26 @@ func crashReportFingerprint(message string) []string { return []string{"{{ default }}", message} } -// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar] -// or, somewhere along the way the "+" in the version tag disappeared: -// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade] -var ( - longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`) - gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618" - gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash -) - -type version struct { - version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep" - tag string // "v1.1.4-rc.1" - commit string // "6aaae618", blank when absent - codename string // "Erbium Earthworm" - runtime string // "go1.12.5" - goos string // "darwin" - goarch string // "amd64" - builder string // "jb@kvin.kastelo.net" - extra []string // "foo", "bar" -} - -func (v version) environment() string { - if v.commit != "" { - return "Development" - } - if strings.Contains(v.tag, "-rc.") { - return "Candidate" - } - if strings.Contains(v.tag, "-") { - return "Beta" - } - return "Stable" -} - -func parseVersion(line string) (version, error) { - m := longVersionRE.FindStringSubmatch(line) - if len(m) == 0 { - return version{}, errors.New("unintelligeble version string") - } - - v := version{ - version: m[1], - codename: m[2], - runtime: m[3], - goos: m[4], - goarch: m[5], - builder: m[6], - } - - // Split the version tag into tag and commit. This is old style - // v1.2.3-something.4+11-g12345678 or newer with just dots - // v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678. - parts := []string{v.version} - if strings.Contains(v.version, "+") { - parts = strings.Split(v.version, "+") - } else { - idxs := gitExtraRE.FindStringIndex(v.version) - if len(idxs) > 0 { - parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]} - } - } - v.tag = parts[0] - if len(parts) > 1 { - fields := gitExtraSepRE.Split(parts[1], -1) - if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") { - v.commit = fields[1][1:] - } - } - - if len(m) >= 8 && m[7] != "" { - tags := strings.Split(m[7], ",") - for i := range tags { - tags[i] = strings.TrimSpace(tags[i]) - } - v.extra = tags - } - - return v, nil -} - -func packet(version version, reportType string) *raven.Packet { +func packet(version build.VersionParts, reportType string) *raven.Packet { pkt := &raven.Packet{ Platform: "go", - Release: version.tag, - Environment: version.environment(), + Release: version.Tag, + Environment: version.Environment(), Tags: raven.Tags{ - raven.Tag{Key: "version", Value: version.version}, - raven.Tag{Key: "tag", Value: version.tag}, - raven.Tag{Key: "codename", Value: version.codename}, - raven.Tag{Key: "runtime", Value: version.runtime}, - raven.Tag{Key: "goos", Value: version.goos}, - raven.Tag{Key: "goarch", Value: version.goarch}, - raven.Tag{Key: "builder", Value: version.builder}, + raven.Tag{Key: "version", Value: version.Version}, + raven.Tag{Key: "tag", Value: version.Tag}, + raven.Tag{Key: "codename", Value: version.Codename}, + raven.Tag{Key: "runtime", Value: version.Runtime}, + raven.Tag{Key: "goos", Value: version.GOOS}, + raven.Tag{Key: "goarch", Value: version.GOARCH}, + raven.Tag{Key: "builder", Value: version.Builder}, raven.Tag{Key: "report_type", Value: reportType}, }, } - if version.commit != "" { - pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit}) + if version.Commit != "" { + pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit}) } - for _, tag := range version.extra { + for _, tag := range version.Extra { pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"}) } return pkt diff --git a/cmd/stcrashreceiver/sentry_test.go b/cmd/stcrashreceiver/sentry_test.go index 9fa30f262..f087641e4 100644 --- a/cmd/stcrashreceiver/sentry_test.go +++ b/cmd/stcrashreceiver/sentry_test.go @@ -12,66 +12,6 @@ import ( "testing" ) -func TestParseVersion(t *testing.T) { - cases := []struct { - longVersion string - parsed version - }{ - { - longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`, - parsed: version{ - version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep", - tag: "v1.1.4-rc.1", - commit: "6aaae618", - codename: "Erbium Earthworm", - runtime: "go1.12.5", - goos: "darwin", - goarch: "amd64", - builder: "jb@kvin.kastelo.net", - }, - }, - { - longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`, - parsed: version{ - version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep", - tag: "v1.1.4-rc.1", - commit: "6aaae618", - codename: "Erbium Earthworm", - runtime: "go1.12.5", - goos: "darwin", - goarch: "amd64", - builder: "jb@kvin.kastelo.net", - extra: []string{"foo", "bar"}, - }, - }, - { - longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`, - parsed: version{ - version: "v1.23.7-dev.26.gdf7b56ae-stversionextra", - tag: "v1.23.7-dev", - commit: "df7b56ae", - codename: "Fermium Flea", - runtime: "go1.20.5", - goos: "darwin", - goarch: "arm64", - builder: "jb@ok.kastelo.net", - extra: []string{"Some Wrapper", "purego", "stnoupgrade"}, - }, - }, - } - - for _, tc := range cases { - v, err := parseVersion(tc.longVersion) - if err != nil { - t.Errorf("%s\nerror: %v\n", tc.longVersion, err) - continue - } - if fmt.Sprint(v) != fmt.Sprint(tc.parsed) { - |