Fix streaming scaling (#8375)

* Disallow multiple streaming connections to the same master agent

* Reject multiple streaming connections quickly without blocking

* Increase timeout for systemd service shutdown to give time to flush the db.

* Optimize page correlation ID to use atomic counter instead of locks

* Reduce contention in global configuration mutex

* Optimize complexity of inserting configuration sections from O(N) to O(1)

* Reduce overhead of clockgettime() by utilizing CLOCK_MONOTONIC_COARSE when applicable.

* Fix unit test compile errors

4 files changed, 37 insertions, 12 deletions

diff --git a/libnetdata/clocks/clocks.c b/libnetdata/clocks/clocks.c
index 161225a9b6..3875b2a255 100644
--- a/libnetdata/clocks/clocks.c
+++ b/libnetdata/clocks/clocks.c
@@ -3,6 +3,7 @@
 #include "../libnetdata.h"
 
 static int clock_boottime_valid = 1;
+static int clock_monotonic_coarse_valid = 1;
 
 #ifndef HAVE_CLOCK_GETTIME
 inline int clock_gettime(clockid_t clk_id, struct timespec *ts) {
@@ -23,6 +24,12 @@ void test_clock_boottime(void) {
         clock_boottime_valid = 0;
 }
 
+void test_clock_monotonic_coarse(void) {
+    struct timespec ts;
+    if(clock_gettime(CLOCK_MONOTONIC_COARSE, &ts) == -1 && errno == EINVAL)
+        clock_monotonic_coarse_valid = 0;
+}
+
 static inline time_t now_sec(clockid_t clk_id) {
     struct timespec ts;
     if(unlikely(clock_gettime(clk_id, &ts) == -1)) {
@@ -69,27 +76,31 @@ inline int now_realtime_timeval(struct timeval *tv) {
 }
 
 inline time_t now_monotonic_sec(void) {
-    return now_sec(CLOCK_MONOTONIC);
+    return now_sec(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
 }
 
 inline usec_t now_monotonic_usec(void) {
-    return now_usec(CLOCK_MONOTONIC);
+    return now_usec(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
 }
 
 inline int now_monotonic_timeval(struct timeval *tv) {
-    return now_timeval(CLOCK_MONOTONIC, tv);
+    return now_timeval(likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC, tv);
 }
 
 inline time_t now_boottime_sec(void) {
-    return now_sec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME : CLOCK_MONOTONIC);
+    return now_sec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME :
+                   likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
 }
 
 inline usec_t now_boottime_usec(void) {
-    return now_usec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME : CLOCK_MONOTONIC);
+    return now_usec(likely(clock_boottime_valid) ? CLOCK_BOOTTIME :
+                    likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC);
 }
 
 inline int now_boottime_timeval(struct timeval *tv) {
-    return now_timeval(likely(clock_boottime_valid) ? CLOCK_BOOTTIME : CLOCK_MONOTONIC, tv);
+    return now_timeval(likely(clock_boottime_valid) ? CLOCK_BOOTTIME :
+                       likely(clock_monotonic_coarse_valid) ? CLOCK_MONOTONIC_COARSE : CLOCK_MONOTONIC,
+                       tv);
 }
 
 inline usec_t timeval_usec(struct timeval *tv) {
diff --git a/libnetdata/clocks/clocks.h b/libnetdata/clocks/clocks.h
index 4af451d60a..7938cb0d36 100644
--- a/libnetdata/clocks/clocks.h
+++ b/libnetdata/clocks/clocks.h
@@ -36,6 +36,12 @@ typedef struct heartbeat {
 #define CLOCK_MONOTONIC CLOCK_REALTIME
 #endif
 
+/* Prefer CLOCK_MONOTONIC_COARSE where available to reduce overhead. It has the same semantics as CLOCK_MONOTONIC */
+#ifndef CLOCK_MONOTONIC_COARSE
+/* fallback to CLOCK_MONOTONIC if not available */
+#define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
+#endif
+
 #ifndef CLOCK_BOOTTIME
 
 #ifdef CLOCK_UPTIME
@@ -43,7 +49,7 @@ typedef struct heartbeat {
 #define CLOCK_BOOTTIME CLOCK_UPTIME
 #else // CLOCK_UPTIME
 /* CLOCK_BOOTTIME falls back to CLOCK_MONOTONIC */
-#define CLOCK_BOOTTIME  CLOCK_MONOTONIC
+#define CLOCK_BOOTTIME  CLOCK_MONOTONIC_COARSE
 #endif // CLOCK_UPTIME
 
 #else // CLOCK_BOOTTIME
@@ -136,6 +142,12 @@ extern int sleep_usec(usec_t usec);
  */
 void test_clock_boottime(void);
 
+/*
+ * When running a binary with CLOCK_MONOTONIC_COARSE defined on a system with a linux kernel older than Linux 2.6.32 the
+ * clock_gettime(2) system call fails with EINVAL. In that case it must fall-back to CLOCK_MONOTONIC.
+ */
+void test_clock_monotonic_coarse(void);
+
 extern collected_number uptime_msec(char *filename);
 
 #endif /* NETDATA_CLOCKS_H */
diff --git a/libnetdata/config/appconfig.c b/libnetdata/config/appconfig.c
index c77cf34ec3..2fb21f1837 100644
--- a/libnetdata/config/appconfig.c
+++ b/libnetdata/config/appconfig.c
@@ -169,12 +169,13 @@ static inline struct section *appconfig_section_create(struct config *root, cons
         error("INTERNAL ERROR: indexing of section '%s', already exists.", co->name);
 
     appconfig_wrlock(root);
-    struct section *co2 = root->sections;
+    struct section *co2 = root->last_section;
     if(co2) {
-        while (co2->next) co2 = co2->next;
         co2->next = co;
+    } else {
+        root->first_section = co;
     }
-    else root->sections = co;
+    root->last_section = co;
     appconfig_unlock(root);
 
     return co;
@@ -678,7 +679,7 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
         }
 
         appconfig_wrlock(root);
-        for(co = root->sections; co ; co = co->next) {
+        for(co = root->first_section; co ; co = co->next) {
             if(!strcmp(co->name, CONFIG_SECTION_GLOBAL)
                || !strcmp(co->name, CONFIG_SECTION_WEB)
                || !strcmp(co->name, CONFIG_SECTION_STATSD)
diff --git a/libnetdata/config/appconfig.h b/libnetdata/config/appconfig.h
index 4688760514..d448205b63 100644
--- a/libnetdata/config/appconfig.h
+++ b/libnetdata/config/appconfig.h
@@ -141,7 +141,8 @@ struct section {
 };
 
 struct config {
-    struct section *sections;
+    struct section *first_section;
+    struct section *last_section; // optimize inserting at the end
     netdata_mutex_t mutex;
     avl_tree_lock index;
 };