Enable support for Netdata Cloud.

This PR merges the feature-branch to make the cloud live. It contains the following work:
Co-authored-by: Andrew Moss <1043609+amoss@users.noreply.github.com(opens in new tab)>
Co-authored-by: Jacek Kolasa <jacek.kolasa@gmail.com(opens in new tab)>
Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud(opens in new tab)>
Co-authored-by: James Mills <prologic@shortcircuit.net.au(opens in new tab)>
Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com(opens in new tab)>
Co-authored-by: Timotej S <6674623+underhood@users.noreply.github.com(opens in new tab)>
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com(opens in new tab)>
* dashboard with new navbars, v1.0-alpha.9: PR #8478
* dashboard v1.0.11: netdata/dashboard#76
Co-authored-by: Jacek Kolasa <jacek.kolasa@gmail.com(opens in new tab)>
* Added installer code to bundle JSON-c if it's not present. PR #8836
Co-authored-by: James Mills <prologic@shortcircuit.net.au(opens in new tab)>
* Fix claiming config PR #8843
* Adds JSON-c as hard dep. for ACLK PR #8838
* Fix SSL renegotiation errors in old versions of openssl. PR #8840. Also - we have a transient problem with opensuse CI so this PR disables them with a commit from @prologic.
Co-authored-by: James Mills <prologic@shortcircuit.net.au(opens in new tab)>
* Fix claiming error handling PR #8850
* Added CI to verify JSON-C bundling code in installer PR #8853
* Make cloud-enabled flag in web/api/v1/info be independent of ACLK build success PR #8866
* Reduce ACLK_STABLE_TIMEOUT from 10 to 3 seconds PR #8871
* remove old-cloud related UI from old dashboard (accessible now via /old suffix) PR #8858
* dashboard v1.0.13 PR #8870
* dashboard v1.0.14 PR #8904
* Provide feedback on proxy setting changes PR #8895
* Change the name of the connect message to update during an ongoing session PR #8927
* Fetch active alarms from alarm_log PR #8944

4 files changed, 170 insertions, 52 deletions

diff --git a/daemon/commands.c b/daemon/commands.c
index 295972e6fe..bf9c30d124 100644
--- a/daemon/commands.c
+++ b/daemon/commands.c
@@ -43,6 +43,8 @@ static cmd_status_t cmd_exit_execute(char *args, char **message);
 static cmd_status_t cmd_fatal_execute(char *args, char **message);
 static cmd_status_t cmd_reload_claiming_state_execute(char *args, char **message);
 static cmd_status_t cmd_reload_labels_execute(char *args, char **message);
+static cmd_status_t cmd_read_config_execute(char *args, char **message);
+static cmd_status_t cmd_write_config_execute(char *args, char **message);
 
 static command_info_t command_info_array[] = {
         {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY},                  // show help menu
@@ -53,6 +55,8 @@ static command_info_t command_info_array[] = {
         {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY},          // exit with fatal error
         {"reload-claiming-state", cmd_reload_claiming_state_execute, CMD_TYPE_ORTHOGONAL}, // reload claiming state
         {"reload-labels", cmd_reload_labels_execute, CMD_TYPE_ORTHOGONAL},   // reload the labels
+        {"read-config", cmd_read_config_execute, CMD_TYPE_CONCURRENT},
+        {"write-config", cmd_write_config_execute, CMD_TYPE_ORTHOGONAL}
 };
 
 /* Mutexes for commands of type CMD_TYPE_ORTHOGONAL */
@@ -185,23 +189,15 @@ static cmd_status_t cmd_reload_claiming_state_execute(char *args, char **message
 {
     (void)args;
     (void)message;
-
-#ifdef DISABLE_CLOUD
-    info("The claiming feature has been disabled");
-    return CMD_STATUS_FAILURE;
-#endif
-#ifndef ENABLE_ACLK
-    info("Cloud functionality is not enabled because of missing dependencies at build-time.");
+#if defined(DISABLE_CLOUD) || !defined(ENABLE_ACLK)
+    info("The claiming feature has been explicitly disabled");
+    *message = strdupz("This agent cannot be claimed, it was built without support for Cloud");
     return CMD_STATUS_FAILURE;
 #endif
-    if (!netdata_cloud_setting) {
-        error("Cannot reload claiming status -> cloud functionality has been disabled");
-        return CMD_STATUS_FAILURE;
-    }
-
     error_log_limit_unlimited();
     info("COMMAND: Reloading Agent Claiming configuration.");
     load_claiming_state();
+    registry_update_cloud_base_url();
     error_log_limit_reset();
     return CMD_STATUS_SUCCESS;
 }
@@ -230,6 +226,76 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
     return CMD_STATUS_SUCCESS;
 }
 
+static cmd_status_t cmd_read_config_execute(char *args, char **message)
+{
+    size_t n = strlen(args);
+    char *separator = strchr(args,'|');
+    if (separator == NULL)
+        return CMD_STATUS_FAILURE;
+    char *separator2 = strchr(separator + 1,'|');
+    if (separator2 == NULL)
+        return CMD_STATUS_FAILURE;
+
+    char *temp = callocz(n + 1, 1);
+    strcpy(temp, args);
+    size_t offset = separator - args;
+    temp[offset] = 0;
+    size_t offset2 = separator2 - args;
+    temp[offset2] = 0;
+
+    const char *conf_file = temp; /* "cloud" is cloud.conf, otherwise netdata.conf */
+    struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;
+
+    char *value = appconfig_get(tmp_config, temp + offset + 1, temp + offset2 + 1, NULL);
+    if (value == NULL)
+    {
+        error("Cannot execute read-config conf_file=%s section=%s / key=%s because no value set", conf_file,
+              temp + offset + 1, temp + offset2 + 1);
+        freez(temp);
+        return CMD_STATUS_FAILURE;
+    }
+    else
+    {
+        (*message) = strdupz(value);
+        freez(temp);
+        return CMD_STATUS_SUCCESS;
+    }
+
+}
+
+static cmd_status_t cmd_write_config_execute(char *args, char **message)
+{
+    UNUSED(message);
+    info("write-config %s", args);
+    size_t n = strlen(args);
+    char *separator = strchr(args,'|');
+    if (separator == NULL)
+        return CMD_STATUS_FAILURE;
+    char *separator2 = strchr(separator + 1,'|');
+    if (separator2 == NULL)
+        return CMD_STATUS_FAILURE;
+    char *separator3 = strchr(separator2 + 1,'|');
+    if (separator3 == NULL)
+        return CMD_STATUS_FAILURE;
+    char *temp = callocz(n + 1, 1);
+    strcpy(temp, args);
+    size_t offset = separator - args;
+    temp[offset] = 0;
+    size_t offset2 = separator2 - args;
+    temp[offset2] = 0;
+    size_t offset3 = separator3 - args;
+    temp[offset3] = 0;
+
+    const char *conf_file = temp; /* "cloud" is cloud.conf, otherwise netdata.conf */
+    struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;
+
+    appconfig_set(tmp_config, temp + offset + 1, temp + offset2 + 1, temp + offset3 + 1);
+    info("write-config conf_file=%s section=%s key=%s value=%s",conf_file, temp + offset + 1, temp + offset2 + 1,
+         temp + offset3 + 1);
+    freez(temp);
+    return CMD_STATUS_SUCCESS;
+}
+
 static void cmd_lock_exclusive(unsigned index)
 {
     (void)index;
@@ -369,9 +435,11 @@ static void schedule_command(uv_work_t *req)
     cmd_ctx->status = execute_command(cmd_ctx->idx, cmd_ctx->args, &cmd_ctx->message);
 }
 
+/* This will alter the state of the command_info_array.cmd_str
+*/
 static void parse_commands(struct command_context *cmd_ctx)
 {
-    char *message = NULL, *pos;
+    char *message = NULL, *pos, *lstrip, *rstrip;
     cmd_t i;
     cmd_status_t status;
 
@@ -381,9 +449,12 @@ static void parse_commands(struct command_context *cmd_ctx)
     for (pos = cmd_ctx->command_string ; isspace(*pos) && ('\0' != *pos) ; ++pos) {;}
     for (i = 0 ; i < CMD_TOTAL_COMMANDS ; ++i) {
         if (!strncmp(pos, command_info_array[i].cmd_str, strlen(command_info_array[i].cmd_str))) {
+            for (lstrip=pos + strlen(command_info_array[i].cmd_str); isspace(*lstrip) && ('\0' != *lstrip); ++lstrip) {;}
+            for (rstrip=lstrip+strlen(lstrip)-1; rstrip>lstrip && isspace(*rstrip); *(rstrip--) = 0 );
+
             cmd_ctx->work.data = cmd_ctx;
             cmd_ctx->idx = i;
-            cmd_ctx->args = pos + strlen(command_info_array[i].cmd_str);
+            cmd_ctx->args = lstrip;
             cmd_ctx->message = NULL;
 
             assert(0 == uv_queue_work(loop, &cmd_ctx->work, schedule_command, after_schedule_command));
diff --git a/daemon/commands.h b/daemon/commands.h
index cb879c2a8c..3b2a1ca9a1 100644
--- a/daemon/commands.h
+++ b/daemon/commands.h
@@ -21,6 +21,8 @@ typedef enum cmd {
     CMD_FATAL,
     CMD_RELOAD_CLAIMING_STATE,
     CMD_RELOAD_LABELS,
+    CMD_READ_CONFIG,
+    CMD_WRITE_CONFIG,
     CMD_TOTAL_COMMANDS
 } cmd_t;
 
diff --git a/daemon/daemon.c b/daemon/daemon.c
index 77448c0e73..7842ecb0be 100644
--- a/daemon/daemon.c
+++ b/daemon/daemon.c
@@ -437,7 +437,7 @@ int become_daemon(int dont_fork, const char *user)
     sched_setscheduler_set();
 
     // Set claiming directory based on user config directory with correct ownership
-    snprintfz(claimingdirectory, FILENAME_MAX, "%s/claim.d", netdata_configured_user_config_dir);
+    snprintfz(claimingdirectory, FILENAME_MAX, "%s/cloud.d", netdata_configured_varlib_dir);
 
     if(user && *user) {
         if(become_user(user, pidfd) != 0) {
diff --git a/daemon/main.c b/daemon/main.c
index b70f3ecb79..ae7bba61fa 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -577,20 +577,7 @@ static void get_netdata_configured_variables() {
     get_system_cpus();
     get_system_pid_max();
 
-    // --------------------------------------------------------------------
-    // Check if the cloud is enabled
-#ifdef DISABLE_CLOUD
-    netdata_cloud_setting = 0;
-#else
-    char *cloud = config_get(CONFIG_SECTION_GLOBAL, "netdata cloud", "coming soon");
-    if (!strcmp(cloud, "coming soon")) {
-        netdata_cloud_setting = 0;          // Note: this flips to 1 after the release
-    } else if (!strcmp(cloud, "enable")) {
-        netdata_cloud_setting = 1;
-    } else if (!strcmp(cloud, "disable")) {
-        netdata_cloud_setting = 0;
-    }
-#endif
+
 }
 
 static void get_system_timezone(void) {
@@ -851,11 +838,41 @@ void set_silencers_filename() {
     silencers_filename = config_get(CONFIG_SECTION_HEALTH, "silencers file", filename);
 }
 
+/* Any config setting that can be accessed without a default value i.e. configget(...,...,NULL) *MUST*
+   be set in this procedure to be called in all the relevant code paths.
+*/
+void post_conf_load(char **user)
+{
+    // --------------------------------------------------------------------
+    // get the user we should run
+
+    // IMPORTANT: this is required before web_files_uid()
+    if(getuid() == 0) {
+        *user = config_get(CONFIG_SECTION_GLOBAL, "run as user", NETDATA_USER);
+    }
+    else {
+        struct passwd *passwd = getpwuid(getuid());
+        *user = config_get(CONFIG_SECTION_GLOBAL, "run as user", (passwd && passwd->pw_name)?passwd->pw_name:"");
+    }
+
+    // --------------------------------------------------------------------
+    // Check if the cloud is enabled
+#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
+    netdata_cloud_setting = 0;
+#else
+    netdata_cloud_setting = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1);
+#endif
+    // This must be set before any point in the code that accesses it. Do not move it from this function.
+    appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);
+}
+
 int main(int argc, char **argv) {
     int i;
     int config_loaded = 0;
     int dont_fork = 0;
     size_t default_stacksize;
+    char *user = NULL;
+
 
     netdata_ready=0;
     // set the name for logging
@@ -918,6 +935,8 @@ int main(int argc, char **argv) {
                     }
                     else {
                         debug(D_OPTIONS, "Configuration loaded from %s.", optarg);
+                        post_conf_load(&user);
+                        load_cloud_conf(1);
                         config_loaded = 1;
                     }
                     break;
@@ -965,6 +984,8 @@ int main(int argc, char **argv) {
                         if(strcmp(optarg, "unittest") == 0) {
                             if(unit_test_buffer()) return 1;
                             if(unit_test_str2ld()) return 1;
+                            // No call to load the config file on this code-path
+                            post_conf_load(&user);
                             get_netdata_configured_variables();
                             default_rrd_update_every = 1;
                             default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
@@ -1065,9 +1086,9 @@ int main(int argc, char **argv) {
                             debug_flags = strtoull(optarg, NULL, 0);
                         }
                         else if(strcmp(optarg, "set") == 0) {
-                            if(optind + 3 > argc) {
-                                fprintf(stderr, "%s", "\nUSAGE: -W set 'section' 'key' 'value'\n\n"
-                                        " Overwrites settings of netdata.conf.\n"
+                            if(optind + 4 > argc) {
+                                fprintf(stderr, "%s", "\nUSAGE: -W set 'conf_file' 'section' 'key' 'value'\n\n"
+                                        " Overwrites settings of netdata.conf or cloud.conf\n"
                                         "\n"
                                         " These options interact with: -c netdata.conf\n"
                                         " If -c netdata.conf is given on the command line,\n"
@@ -1076,21 +1097,24 @@ int main(int argc, char **argv) {
                                         " If -c netdata.conf is given after (or missing)\n"
                                         " -W set... the user cannot overwrite the command line\n"
                                         " parameters."
+                                        " conf_file can be \"cloud\" or \"netdata\".\n"
                                         "\n"
                                 );
                                 return 1;
                             }
-                            const char *section = argv[optind];
-                            const char *key = argv[optind + 1];
-                            const char *value = argv[optind + 2];
-                            optind += 3;
+                            const char *conf_file = argv[optind]; /* "cloud" is cloud.conf, otherwise netdata.conf */
+                            struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;
+                            const char *section = argv[optind + 1];
+                            const char *key = argv[optind + 2];
+                            const char *value = argv[optind + 3];
+                            optind += 4;
 
                             // set this one as the default
                             // only if it is not already set in the config file
                             // so the caller can use -c netdata.conf before or
                             // after this parameter to prevent or allow overwriting
                             // variables at netdata.conf
-                            config_set_default(section, key,  value);
+                            appconfig_set_default(tmp_config, section, key,  value);
 
                             // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value);
                         }
@@ -1109,6 +1133,7 @@ int main(int argc, char **argv) {
                             if(!config_loaded) {
                                 fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n");
                                 load_netdata_conf(NULL, 0);
+                                post_conf_load(&user);
                             }
 
                             get_netdata_configured_variables();
@@ -1120,6 +1145,37 @@ int main(int argc, char **argv) {
                             printf("%s\n", value);
                             return 0;
                         }
+                        else if(strcmp(optarg, "get2") == 0) {
+                            if(optind + 4 > argc) {
+                                fprintf(stderr, "%s", "\nUSAGE: -W get2 'conf_file' 'section' 'key' 'value'\n\n"
+                                        " Prints settings of netdata.conf or cloud.conf\n"
+                                        "\n"
+                                        " These options interact with: -c netdata.conf\n"
+                                        " -c netdata.conf has to be given before -W get2.\n"
+                                        " conf_file can be \"cloud\" or \"netdata\".\n"
+                                        "\n"
+                                );
+                                return 1;
+                            }
+
+                            if(!config_loaded) {
+                                fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n");
+                                load_netdata_conf(NULL, 0);
+                                post_conf_load(&user);
+                                load_cloud_conf(1);
+                            }
+
+                            get_netdata_configured_variables();
+
+                            const char *conf_file = argv[optind]; /* "cloud" is cloud.conf, otherwise netdata.conf */
+                            struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;
+                            const char *section = argv[optind + 1];
+                            const char *key = argv[optind + 2];
+                            const char *def = argv[optind + 3];
+                            const char *value = appconfig_get(tmp_config, section, key, def);
+                            printf("%s\n", value);
+                            return 0;
+                        }
                         else if(strncmp(optarg, claim_string, strlen(claim_string)) == 0) {
                             /* will trigger a claiming attempt when the agent is initialized */
                             claiming_pending_arguments = optarg + strlen(claim_string);
@@ -1149,7 +1205,11 @@ int main(int argc, char **argv) {
 #endif
 
     if(!config_loaded)
+    {
         load_netdata_conf(NULL, 0);
+        post_conf_load(&user);
+        load_cloud_conf(0);
+    }
 
 
     // ------------------------------------------------------------------------
@@ -1179,8 +1239,6 @@ int main(int argc, char **argv) {
             fatal("Cannot cd to '%s'", netdata_configured_user_config_dir);
     }
 
-    char *user = NULL;
-
     {
         // --------------------------------------------------------------------
         // get the debugging flags from the configuration file
@@ -1247,19 +1305,6 @@ int main(int argc, char **argv) {
 
 
         // --------------------------------------------------------------------
-        // get the user we should run
-
-        // IMPORTANT: this is required before web_files_uid()
-        if(getuid() == 0) {
-            user = config_get(CONFIG_SECTION_GLOBAL, "run as user", NETDATA_USER);
-        }
-        else {
-            struct passwd *passwd = getpwuid(getuid());
-            user = config_get(CONFIG_SECTION_GLOBAL, "run as user", (passwd && passwd->pw_name)?passwd->pw_name:"");
-        }
-
-
-        // --------------------------------------------------------------------
         // create the listening sockets
 
         web_client_api_v1_init();