lib/connections: Refactor

1. Removes separate relay lists and relay clients/services, just makes it a listen address
2. Easier plugging-in of other transports
3. Allows "hot" disabling and enabling NAT services
4. Allows "hot" listen address changes
5. Changes listen address list with a preferable "default" value just like for discovery
6. Debounces global discovery announcements as external addresses change (which it might alot upon starting)
7. Stops this whole "pick other peers relay by latency". This information is no longer available,
   but I don't think it matters as most of the time other peer only has one relay.
8. Rename ListenAddress to ListenAddresses, as well as in javascript land.
9. Stop serializing deprecated values to JSON

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/2982

45 files changed, 1683 insertions, 1707 deletions

diff --git a/cmd/stfinddevice/main.go b/cmd/stfinddevice/main.go
index 939a3c0f39..c4ee8ddfbb 100644
--- a/cmd/stfinddevice/main.go
+++ b/cmd/stfinddevice/main.go
@@ -49,9 +49,8 @@ func main() {
 }
 
 type checkResult struct {
-	server string
-	direct []string
-	relays []discover.Relay
+	server    string
+	addresses []string
 	error
 }
 
@@ -76,17 +75,14 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 		if res.error != nil {
 			fmt.Println("  " + res.error.Error())
 		}
-		for _, addr := range res.direct {
+		for _, addr := range res.addresses {
 			fmt.Println("  address:", addr)
 		}
-		for _, rel := range res.relays {
-			fmt.Printf("  relay: %s (%d ms)\n", rel.URL, rel.Latency)
-		}
 	}
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
 	if err != nil {
 		return checkResult{error: err}
 	}
@@ -98,8 +94,8 @@ func checkServer(deviceID protocol.DeviceID, server string) checkResult {
 	})
 
 	go func() {
-		direct, relays, err := disco.Lookup(deviceID)
-		res <- checkResult{direct: direct, relays: relays, error: err}
+		addresses, err := disco.Lookup(deviceID)
+		res <- checkResult{addresses: addresses, error: err}
 	}()
 
 	return <-res
diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go
index 5fa63e10da..3ae5537d97 100644
--- a/cmd/syncthing/gui.go
+++ b/cmd/syncthing/gui.go
@@ -35,7 +35,6 @@ import (
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/stats"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -51,21 +50,21 @@ var (
 )
 
 type apiService struct {
-	id              protocol.DeviceID
-	cfg             configIntf
-	httpsCertFile   string
-	httpsKeyFile    string
-	assetDir        string
-	themes          []string
-	model           modelIntf
-	eventSub        events.BufferedSubscription
-	discoverer      discover.CachingMux
-	relayService    relay.Service
-	fss             *folderSummaryService
-	systemConfigMut sync.Mutex    // serializes posts to /rest/system/config
-	stop            chan struct{} // signals intentional stop
-	configChanged   chan struct{} // signals intentional listener close due to config change
-	started         chan struct{} // signals startup complete, for testing only
+	id                 protocol.DeviceID
+	cfg                configIntf
+	httpsCertFile      string
+	httpsKeyFile       string
+	assetDir           string
+	themes             []string
+	model              modelIntf
+	eventSub           events.BufferedSubscription
+	discoverer         discover.CachingMux
+	connectionsService connectionsIntf
+	fss                *folderSummaryService
+	systemConfigMut    sync.Mutex    // serializes posts to /rest/system/config
+	stop               chan struct{} // signals intentional stop
+	configChanged      chan struct{} // signals intentional listener close due to config change
+	started            chan struct{} // signals startup complete, for testing only
 
 	listener    net.Listener
 	listenerMut sync.Mutex
@@ -113,25 +112,30 @@ type configIntf interface {
 	Folders() map[string]config.FolderConfiguration
 	Devices() map[protocol.DeviceID]config.DeviceConfiguration
 	Save() error
+	ListenAddresses() []string
 }
 
-func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, eventSub events.BufferedSubscription, discoverer discover.CachingMux, relayService relay.Service, errors, systemLog logger.Recorder) (*apiService, error) {
+type connectionsIntf interface {
+	Status() map[string]interface{}
+}
+
+func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, eventSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connectionsIntf, errors, systemLog logger.Recorder) (*apiService, error) {
 	service := &apiService{
-		id:              id,
-		cfg:             cfg,
-		httpsCertFile:   httpsCertFile,
-		httpsKeyFile:    httpsKeyFile,
-		assetDir:        assetDir,
-		model:           m,
-		eventSub:        eventSub,
-		discoverer:      discoverer,
-		relayService:    relayService,
-		systemConfigMut: sync.NewMutex(),
-		stop:            make(chan struct{}),
-		configChanged:   make(chan struct{}),
-		listenerMut:     sync.NewMutex(),
-		guiErrors:       errors,
-		systemLog:       systemLog,
+		id:                 id,
+		cfg:                cfg,
+		httpsCertFile:      httpsCertFile,
+		httpsKeyFile:       httpsKeyFile,
+		assetDir:           assetDir,
+		model:              m,
+		eventSub:           eventSub,
+		discoverer:         discoverer,
+		connectionsService: connectionsService,
+		systemConfigMut:    sync.NewMutex(),
+		stop:               make(chan struct{}),
+		configChanged:      make(chan struct{}),
+		listenerMut:        sync.NewMutex(),
+		guiErrors:          errors,
+		systemLog:          systemLog,
 	}
 
 	seen := make(map[string]struct{})
@@ -825,18 +829,9 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 		res["discoveryMethods"] = discoMethods
 		res["discoveryErrors"] = discoErrors
 	}
-	if s.relayService != nil {
-		res["relaysEnabled"] = true
-		relayClientStatus := make(map[string]bool)
-		relayClientLatency := make(map[string]int)
-		for _, relay := range s.relayService.Relays() {
-			latency, ok := s.relayService.RelayStatus(relay)
-			relayClientStatus[relay] = ok
-			relayClientLatency[relay] = int(latency / time.Millisecond)
-		}
-		res["relayClientStatus"] = relayClientStatus
-		res["relayClientLatency"] = relayClientLatency
-	}
+
+	res["connectionServiceStatus"] = s.connectionsService.Status()
+
 	cpuUsageLock.RLock()
 	var cpusum float64
 	for _, p := range cpuUsagePercent {
diff --git a/cmd/syncthing/gui_test.go b/cmd/syncthing/gui_test.go
index 45d545ca20..e5ed2d07d2 100644
--- a/cmd/syncthing/gui_test.go
+++ b/cmd/syncthing/gui_test.go
@@ -462,13 +462,13 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 	assetDir := "../../gui"
 	eventSub := new(mockedEventSub)
 	discoverer := new(mockedCachingMux)
-	relayService := new(mockedRelayService)
+	connections := new(mockedConnections)
 	errorLog := new(mockedLoggerRecorder)
 	systemLog := new(mockedLoggerRecorder)
 
 	// Instantiate the API service
 	svc, err := newAPIService(protocol.LocalDeviceID, cfg, httpsCertFile, httpsKeyFile, assetDir, model,
-		eventSub, discoverer, relayService, errorLog, systemLog)
+		eventSub, discoverer, connections, errorLog, systemLog)
 	if err != nil {
 		return "", err
 	}
diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go
index 0d6dd6ff3a..a34a39249d 100644
--- a/cmd/syncthing/main.go
+++ b/cmd/syncthing/main.go
@@ -17,7 +17,6 @@ import (
 	"net"
 	"net/http"
 	_ "net/http/pprof"
-	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -38,19 +37,13 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/model"
-	"github.com/syncthing/syncthing/lib/nat"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/relay"
 	"github.com/syncthing/syncthing/lib/symlinks"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/syncthing/syncthing/lib/util"
 
-	// Registers NAT service providers
-	_ "github.com/syncthing/syncthing/lib/pmp"
-	_ "github.com/syncthing/syncthing/lib/upnp"
-
 	"github.com/thejerf/suture"
 )
 
@@ -696,40 +689,6 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	mainService.Add(m)
 
-	// Start NAT service
-	var natService *nat.Service
-	var mappings []*nat.Mapping
-	if opts.NATEnabled {
-		natService = nat.NewService(myID, cfg)
-		for _, addrStr := range opts.ListenAddress {
-			uri, err := url.Parse(addrStr)
-			if err != nil {
-				l.Fatalf("Failed to parse listen address %s: %v", addrStr, err)
-			}
-
-			if uri.Scheme == "tcp" || uri.Scheme == "tcp4" {
-				addr, err := net.ResolveTCPAddr(uri.Scheme, uri.Host)
-				if err != nil {
-					l.Fatalln("Bad listen address:", err)
-				}
-				if addr.Port == 0 {
-					l.Fatalf("Listen address %s: invalid port", uri)
-				}
-
-				mappings = append(mappings, natService.NewMapping(nat.TCP, addr.IP, addr.Port))
-			}
-		}
-		mainService.Add(natService)
-	}
-
-	// Start relay management
-
-	var relayService relay.Service
-	if opts.RelaysEnabled {
-		relayService = relay.NewService(cfg, tlsCfg)
-		mainService.Add(relayService)
-	}
-
 	// Start discovery
 
 	cachedDiscovery := discover.NewCachingMux()
@@ -737,13 +696,13 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	// Start connection management
 
-	connectionService := connections.NewConnectionService(cfg, myID, m, tlsCfg, cachedDiscovery, mappings, relayService, bepProtocolName, tlsDefaultCommonName, lans)
-	mainService.Add(connectionService)
+	connectionsService := connections.NewService(cfg, myID, m, tlsCfg, cachedDiscovery, bepProtocolName, tlsDefaultCommonName, lans)
+	mainService.Add(connectionsService)
 
 	if cfg.Options().GlobalAnnEnabled {
 		for _, srv := range cfg.GlobalDiscoveryServers() {
 			l.Infoln("Using discovery server", srv)
-			gd, err := discover.NewGlobal(srv, cert, connectionService, relayService)
+			gd, err := discover.NewGlobal(srv, cert, connectionsService)
 			if err != nil {
 				l.Warnln("Global discovery:", err)
 				continue
@@ -758,14 +717,14 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	if cfg.Options().LocalAnnEnabled {
 		// v4 broadcasts
-		bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), connectionService, relayService)
+		bcd, err := discover.NewLocal(myID, fmt.Sprintf(":%d", cfg.Options().LocalAnnPort), connectionsService)
 		if err != nil {
 			l.Warnln("IPv4 local discovery:", err)
 		} else {
 			cachedDiscovery.Add(bcd, 0, 0, ipv4LocalDiscoveryPriority)
 		}
 		// v6 multicasts
-		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, connectionService, relayService)
+		mcd, err := discover.NewLocal(myID, cfg.Options().LocalAnnMCAddr, connectionsService)
 		if err != nil {
 			l.Warnln("IPv6 local discovery:", err)
 		} else {
@@ -775,7 +734,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	// GUI
 
-	setupGUI(mainService, cfg, m, apiSub, cachedDiscovery, relayService, errors, systemLog, runtimeOptions)
+	setupGUI(mainService, cfg, m, apiSub, cachedDiscovery, connectionsService, errors, systemLog, runtimeOptions)
 
 	if runtimeOptions.cpuProfile {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
@@ -957,7 +916,7 @@ func startAuditing(mainService *suture.Supervisor) {
 	l.Infoln("Audit log in", auditFile)
 }
 
-func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub events.BufferedSubscription, discoverer discover.CachingMux, relayService relay.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
+func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService *connections.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
 	guiCfg := cfg.GUI()
 
 	if !guiCfg.Enabled {
@@ -968,7 +927,7 @@ func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Mode
 		l.Warnln("Insecure admin access is enabled.")
 	}
 
-	api, err := newAPIService(myID, cfg, locations[locHTTPSCertFile], locations[locHTTPSKeyFile], runtimeOptions.assetDir, m, apiSub, discoverer, relayService, errors, systemLog)
+	api, err := newAPIService(myID, cfg, locations[locHTTPSCertFile], locations[locHTTPSKeyFile], runtimeOptions.assetDir, m, apiSub, discoverer, connectionsService, errors, systemLog)
 	if err != nil {
 		l.Fatalln("Cannot start GUI:", err)
 	}
@@ -1017,7 +976,15 @@ func defaultConfig(myName string) config.Configuration {
 	if err != nil {
 		l.Fatalln("get free port (BEP):", err)
 	}
-	newCfg.Options.ListenAddress = []string{fmt.Sprintf("tcp://0.0.0.0:%d", port)}
+	if port == 22000 {
+		newCfg.Options.ListenAddresses = []string{"default"}
+	} else {
+		newCfg.Options.ListenAddresses = []string{
+			fmt.Sprintf("tcp://%s", net.JoinHostPort("0.0.0.0", strconv.Itoa(port))),
+			"dynamic+https://relays.syncthing.net/endpoint",
+		}
+	}
+
 	return newCfg
 }
 
diff --git a/cmd/syncthing/mocked_config_test.go b/cmd/syncthing/mocked_config_test.go
index c0bc6366c0..d4e734a739 100644
--- a/cmd/syncthing/mocked_config_test.go
+++ b/cmd/syncthing/mocked_config_test.go
@@ -19,6 +19,10 @@ func (c *mockedConfig) GUI() config.GUIConfiguration {
 	return c.gui
 }
 
+func (c *mockedConfig) ListenAddresses() []string {
+	return nil
+}
+
 func (c *mockedConfig) Raw() config.Configuration {
 	return config.Configuration{}
 }
diff --git a/cmd/syncthing/mocked_connections_test.go b/cmd/syncthing/mocked_connections_test.go
new file mode 100644
index 0000000000..b4b8ffe08d
--- /dev/null
+++ b/cmd/syncthing/mocked_connections_test.go
@@ -0,0 +1,13 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+type mockedConnections struct{}
+
+func (m *mockedConnections) Status() map[string]interface{} {
+	return nil
+}
diff --git a/cmd/syncthing/mocked_discovery_test.go b/cmd/syncthing/mocked_discovery_test.go
index 4d8b9190dd..8f8a977ab2 100644
--- a/cmd/syncthing/mocked_discovery_test.go
+++ b/cmd/syncthing/mocked_discovery_test.go
@@ -26,8 +26,8 @@ func (m *mockedCachingMux) Stop() {
 
 // from events.Finder
 
-func (m *mockedCachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, relays []discover.Relay, err error) {
-	return nil, nil, nil
+func (m *mockedCachingMux) Lookup(deviceID protocol.DeviceID) (direct []string, err error) {
+	return nil, nil
 }
 
 func (m *mockedCachingMux) Error() error {
diff --git a/cmd/syncthing/mocked_relay_test.go b/cmd/syncthing/mocked_relay_test.go
deleted file mode 100644
index 211bfdbd81..0000000000
--- a/cmd/syncthing/mocked_relay_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"crypto/tls"
-	"time"
-)
-
-type mockedRelayService struct{}
-
-// from suture.Service
-
-func (s *mockedRelayService) Serve() {
-	select {}
-}
-
-func (s *mockedRelayService) Stop() {
-}
-
-// from relay.Service
-
-func (s *mockedRelayService) Accept() *tls.Conn {
-	return nil
-}
-
-func (s *mockedRelayService) Relays() []string {
-	return nil
-}
-
-func (s *mockedRelayService) RelayStatus(uri string) (time.Duration, bool) {
-	return 0, false
-}
diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go
index bb20c9d2b4..fc64ddbda1 100644
--- a/cmd/syncthing/usage_report.go
+++ b/cmd/syncthing/usage_report.go
@@ -16,6 +16,7 @@ import (
 	"net/http"
 	"runtime"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
@@ -203,16 +204,16 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
 	}
 
 	defaultRelayServers, otherRelayServers := 0, 0
-	for _, addr := range cfg.Options().RelayServers {
-		switch addr {
-		case "dynamic+https://relays.syncthing.net/endpoint":
+	for _, addr := range cfg.ListenAddresses() {
+		switch {
+		case addr == "dynamic+https://relays.syncthing.net/endpoint":
 			defaultRelayServers++
-		default:
+		case strings.HasPrefix(addr, "relay://") || strings.HasPrefix(addr, "dynamic+http"):
 			otherRelayServers++
 		}
 	}
 	res["relays"] = map[string]interface{}{
-		"enabled":        cfg.Options().RelaysEnabled,
+		"enabled":        defaultRelayServers+otherAnnounceServers > 0,
 		"defaultServers": defaultRelayServers,
 		"otherServers":   otherRelayServers,
 	}
diff --git a/cmd/syncthing/verboseservice.go b/cmd/syncthing/verboseservice.go
index 1852920d66..095b32112e 100644
--- a/cmd/syncthing/verboseservice.go
+++ b/cmd/syncthing/verboseservice.go
@@ -8,7 +8,6 @@ package main
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/syncthing/syncthing/lib/events"
 )
@@ -147,17 +146,12 @@ func (s *verboseService) formatEvent(ev events.Event) string {
 		data := ev.Data.(map[string]string)
 		device := data["device"]
 		return fmt.Sprintf("Device %v was resumed", device)
-	case events.ExternalPortMappingChanged:
+	case events.ListenAddressesChanged:
 		data := ev.Data.(map[string]interface{})
-		protocol := data["protocol"]
-		local := data["local"]
-		added := data["added"]
-		removed := data["removed"]
-		return fmt.Sprintf("External port mapping changed; protocol: %s, local: %s, added: %s, removed: %s", protocol, local, added, removed)
-	case events.RelayStateChanged:
-		data := ev.Data.(map[string][]string)
-		newRelays := data["new"]
-		return fmt.Sprintf("Relay state changed; connected relay(s) are %s.", strings.Join(newRelays, ", "))
+		address := data["address"]
+		lan := data["lan"]
+		wan := data["wan"]
+		return fmt.Sprintf("Listen address %s resolution has changed: lan addresses: %s wan addresses: %s", address, lan, wan)
 	case events.LoginAttempt:
 		data := ev.Data.(map[string]interface{})
 		username := data["username"].(string)
diff --git a/gui/default/index.html b/gui/default/index.html
index 33b7682c05..d0793e03c2 100755
--- a/gui/default/index.html
+++ b/gui/default/index.html
@@ -430,6 +430,19 @@
                     <th><span class="fa fa-fw fa-tachometer"></span>&nbsp;<span translate>CPU Utilization</span></th>
                     <td class="text-right">{{system.cpuPercent | alwaysNumber | natural:1}}%</td>
                   </tr>
+                  <tr>
+                    <th><span class="fa fa-fw fa-sitemap"></span>&nbsp;<span translate>Listeners</span></th>
+                    <td class="text-right">
+                      <span ng-if="listenersFailed.length == 0" class="data text-success">
+                        <span>{{listenersTotal}}/{{listenersTotal}}</span>
+                      </span>
+                      <span ng-if="listenersFailed.length != 0" class="data" ng-class="{'text-danger': listenersFailed.length == listenersTotal}">
+                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{listenersFailed.join('<br>\n')}}">
+                          {{listenersTotal-listenersFailed.length}}/{{listenersTotal}}
+                        </span>
+                      </span>
+                    </td>
+                  </tr>
                   <tr ng-if="system.discoveryEnabled">
                     <th><span class="fa fa-fw fa-map-signs"></span>&nbsp;<span translate>Discovery</span></th>
                     <td class="text-right">
@@ -443,19 +456,6 @@
                       </span>
                     </td>
                   </tr>
-                  <tr ng-if="system.relaysEnabled">
-                    <th><span class="fa fa-fw fa-sitemap"></span>&nbsp;<span translate>Relays</span></th>
-                    <td class="text-right">
-                      <span ng-if="relaysFailed.length == 0" class="data text-success">
-                        <span>{{relaysTotal}}/{{relaysTotal}}</span>
-                      </span>
-                      <span ng-if="relaysFailed.length != 0" class="data" ng-class="{'text-danger': relaysFailed.length == relaysTotal}">
-                        <span popover data-trigger="hover" data-placement="bottom" data-html="true" data-content="{{relaysFailed.join('<br>\n')}}">
-                          {{relaysTotal-relaysFailed.length}}/{{relaysTotal}}
-                        </span>
-                      </span>
-                    </td>
-                  </tr>
                   <tr>
                     <th><span class="fa fa-fw fa-clock-o"></span>&nbsp;<span translate>Uptime</span></th>