Add system info streaming (#5996)

* Add system info streaming

* Fix segmentation fault in unit testing

5 files changed, 193 insertions, 27 deletions

diff --git a/daemon/main.c b/daemon/main.c
index a6d6a7e7ae..fe9e9963c1 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -650,7 +650,7 @@ static int load_netdata_conf(char *filename, char overwrite_used) {
     return ret;
 }
 
-int get_system_info () {
+int get_system_info(struct rrdhost_system_info *system_info) {
     char *script;
     script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("system-info.sh") + 2));
     sprintf(script, "%s/%s", netdata_configured_primary_plugins_dir, "system-info.sh");
@@ -679,10 +679,15 @@ int get_system_info () {
                     (*newline) = '\0';
                 }
                 char n[51], v[101];
-                snprintfz(n,50,"%s",name);
-                snprintfz(v,101,"%s",value);
-                info("%s=%s", n, v);
-                setenv(n, v, 1);
+                snprintfz(n, 50,"%s",name);
+                snprintfz(v, 101,"%s",value);
+                if(unlikely(rrdhost_set_system_info_variable(system_info, n, v))) {
+                    info("Unexpected environment variable %s=%s", n, v);
+                }
+                else {
+                    info("%s=%s", n, v);
+                    setenv(n, v, 1);
+                }
             }
         }
         mypclose(fp, command_pid);
@@ -844,7 +849,7 @@ int main(int argc, char **argv) {
                             default_rrd_update_every = 1;
                             default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
                             default_health_enabled = 0;
-                            rrd_init("unittest");
+                            rrd_init("unittest", NULL);
                             default_rrdpush_enabled = 0;
                             if(run_all_mockup_tests()) return 1;
                             if(unit_test_storage()) return 1;
@@ -1093,8 +1098,10 @@ int main(int argc, char **argv) {
 
     // initialize the log files
     open_all_log_files();
+
 	netdata_anonymous_statistics_enabled=-1;
-    if (get_system_info() == 0) send_statistics("START","-", "-");
+    struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
+    if (get_system_info(system_info) == 0) send_statistics("START","-", "-");
 
 #ifdef NETDATA_INTERNAL_CHECKS
     if(debug_flags != 0) {
@@ -1129,14 +1136,14 @@ int main(int argc, char **argv) {
     // ------------------------------------------------------------------------
     // initialize rrd, registry, health, rrdpush, etc.
 
-    rrd_init(netdata_configured_hostname);
+    rrd_init(netdata_configured_hostname, system_info);
+    rrdhost_system_info_free(system_info);
 
     // ------------------------------------------------------------------------
     // enable log flood protection
 
     error_log_limit_reset();
 
-
     // ------------------------------------------------------------------------
     // spawn the threads
 
diff --git a/database/rrd.h b/database/rrd.h
index 1572e62ab3..ab3da6ee48 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -490,6 +490,22 @@ typedef struct alarm_log {
 // ----------------------------------------------------------------------------
 // RRD HOST
 
+struct rrdhost_system_info {
+    char *os_name;
+    char *os_id;
+    char *os_id_like;
+    char *os_version;
+    char *os_version_id;
+    char *os_detection;
+    char *kernel_name;
+    char *kernel_version;
+    char *architecture;
+    char *virtualization;
+    char *virt_detection;
+    char *container;
+    char *container_detection;
+};
+
 struct rrdhost {
     avl avl;                                        // the index of hosts
 
@@ -520,6 +536,8 @@ struct rrdhost {
     char *program_name;                             // the program name that collects metrics for this host
     char *program_version;                          // the program version that collects metrics for this host
 
+    struct rrdhost_system_info *system_info;        // information collected from the host environment
+
     // ------------------------------------------------------------------------
     // streaming of data to remote hosts - rrdpush
 
@@ -634,7 +652,7 @@ extern netdata_rwlock_t rrd_rwlock;
 extern size_t rrd_hosts_available;
 extern time_t rrdhost_free_orphan_time;
 
-extern void rrd_init(char *hostname);
+extern void rrd_init(char *hostname, struct rrdhost_system_info *system_info);
 
 extern RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash);
 extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash);
@@ -656,8 +674,12 @@ extern RRDHOST *rrdhost_find_or_create(
         , char *rrdpush_destination
         , char *rrdpush_api_key
         , char *rrdpush_send_charts_matching
+        , struct rrdhost_system_info *system_info
 );
 
+extern int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value);
+extern struct rrdhost_system_info *rrdhost_system_info_dup(struct rrdhost_system_info *system_info);
+
 #if defined(NETDATA_INTERNAL_CHECKS) && defined(NETDATA_VERIFY_LOCKS)
 extern void __rrdhost_check_wrlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);
 extern void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);
@@ -714,6 +736,7 @@ extern void rrdhost_save_all(void);
 extern void rrdhost_cleanup_all(void);
 
 extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected);
+extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info);
 extern void rrdhost_free(RRDHOST *host);
 extern void rrdhost_save_charts(RRDHOST *host);
 extern void rrdhost_delete_charts(RRDHOST *host);
diff --git a/database/rrdhost.c b/database/rrdhost.c
index 5f0f8ac645..de0366267c 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -122,6 +122,7 @@ RRDHOST *rrdhost_create(const char *hostname,
                         char *rrdpush_destination,
                         char *rrdpush_api_key,
                         char *rrdpush_send_charts_matching,
+                        struct rrdhost_system_info *system_info,
                         int is_localhost
 ) {
     debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);
@@ -157,6 +158,8 @@ RRDHOST *rrdhost_create(const char *hostname,
     host->program_version = strdupz((program_version && *program_version)?program_version:"unknown");
     host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
 
+    host->system_info = rrdhost_system_info_dup(system_info);
+
     avl_init_lock(&(host->rrdset_root_index),      rrdset_compare);
     avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name);
     avl_init_lock(&(host->rrdfamily_root_index),   rrdfamily_compare);
@@ -332,6 +335,7 @@ RRDHOST *rrdhost_find_or_create(
         , char *rrdpush_destination
         , char *rrdpush_api_key
         , char *rrdpush_send_charts_matching
+        , struct rrdhost_system_info *system_info
 ) {
     debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);
 
@@ -355,6 +359,7 @@ RRDHOST *rrdhost_find_or_create(
                 , rrdpush_destination
                 , rrdpush_api_key
                 , rrdpush_send_charts_matching
+                , system_info
                 , 0
         );
     }
@@ -439,7 +444,7 @@ restart_after_removal:
 // ----------------------------------------------------------------------------
 // RRDHOST global / startup initialization
 
-void rrd_init(char *hostname) {
+void rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
     rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time);
     gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", gap_when_lost_iterations_above);
     if (gap_when_lost_iterations_above < 1)
@@ -468,6 +473,7 @@ void rrd_init(char *hostname) {
             , default_rrdpush_destination
             , default_rrdpush_api_key
             , default_rrdpush_send_charts_matching
+            , system_info
             , 1
     );
     rrd_unlock();
@@ -513,6 +519,27 @@ void __rrd_check_wrlock(const char *file, const char *function, const unsigned l
 // ----------------------------------------------------------------------------
 // RRDHOST - free
 
+void rrdhost_system_info_free(struct rrdhost_system_info *system_info) {
+    info("SYSTEM_INFO: free %p", system_info);
+
+    if(likely(system_info)) {
+        freez(system_info->os_name);
+        freez(system_info->os_id);
+        freez(system_info->os_id_like);
+        freez(system_info->os_version);
+        freez(system_info->os_version_id);
+        freez(system_info->os_detection);
+        freez(system_info->kernel_name);
+        freez(system_info->kernel_version);
+        freez(system_info->architecture);
+        freez(system_info->virtualization);
+        freez(system_info->virt_detection);
+        freez(system_info->container);
+        freez(system_info->container_detection);
+        freez(system_info);
+    }
+}
+
 void rrdhost_free(RRDHOST *host) {
     if(!host) return;
 
@@ -573,6 +600,7 @@ void rrdhost_free(RRDHOST *host) {
     freez((void *)host->timezone);
     freez(host->program_version);
     freez(host->program_name);
+    rrdhost_system_info_free(host->system_info);
     freez(host->cache_dir);
     freez(host->varlib_dir);
     freez(host->rrdpush_send_api_key);
@@ -744,3 +772,73 @@ restart_after_removal:
         }
     }
 }
+
+// ----------------------------------------------------------------------------
+// RRDHOST - set system info from environment variables
+
+int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value) {
+    if(!strcmp(name, "NETDATA_SYSTEM_OS_NAME")){
+        system_info->os_name = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_OS_ID")){
+        system_info->os_id = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_OS_ID_LIKE")){
+        system_info->os_id_like = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_OS_VERSION")){
+        system_info->os_version = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_OS_VERSION_ID")){
+        system_info->os_version_id = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_OS_DETECTION")){
+        system_info->os_detection = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_KERNEL_NAME")){
+        system_info->kernel_name = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_KERNEL_VERSION")){
+        system_info->kernel_version = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_ARCHITECTURE")){
+        system_info->architecture = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_VIRTUALIZATION")){
+        system_info->virtualization = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_VIRT_DETECTION")){
+        system_info->virt_detection = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_CONTAINER")){
+        system_info->container = strdupz(value);
+    }
+    else if(!strcmp(name, "NETDATA_SYSTEM_CONTAINER_DETECTION")){
+        system_info->container_detection = strdupz(value);
+    }
+    else return 1;
+
+    return 0;
+}
+
+struct rrdhost_system_info *rrdhost_system_info_dup(struct rrdhost_system_info *system_info) {
+    struct rrdhost_system_info *ret = callocz(1, sizeof(struct rrdhost_system_info));
+
+    if(likely(system_info)) {
+        ret->os_name = strdupz(system_info->os_name);
+        ret->os_id = strdupz(system_info->os_id);
+        ret->os_id_like = strdupz(system_info->os_id_like);
+        ret->os_version = strdupz(system_info->os_version);
+        ret->os_version_id = strdupz(system_info->os_version_id);
+        ret->os_detection = strdupz(system_info->os_detection);
+        ret->kernel_name = strdupz(system_info->kernel_name);
+        ret->kernel_version = strdupz(system_info->kernel_version);
+        ret->architecture = strdupz(system_info->architecture);
+        ret->virtualization = strdupz(system_info->virtualization);
+        ret->virt_detection = strdupz(system_info->virt_detection);
+        ret->container = strdupz(system_info->container);
+        ret->container_detection = strdupz(system_info->container_detection);
+    }
+
+    return ret;
+}
diff --git a/streaming/rrdpush.c b/streaming/rrdpush.c
index acf69ffd33..955a193c42 100644
--- a/streaming/rrdpush.c
+++ b/streaming/rrdpush.c
@@ -445,7 +445,21 @@ static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_po
     #define HTTP_HEADER_SIZE 8192
     char http[HTTP_HEADER_SIZE + 1];
     snprintfz(http, HTTP_HEADER_SIZE,
-            "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s HTTP/1.1\r\n"
+            "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s"
+                    "&NETDATA_SYSTEM_OS_NAME=%s"
+                    "&NETDATA_SYSTEM_OS_ID=%s"
+                    "&NETDATA_SYSTEM_OS_ID_LIKE=%s"
+                    "&NETDATA_SYSTEM_OS_VERSION=%s"
+                    "&NETDATA_SYSTEM_OS_VERSION_ID=%s"
+                    "&NETDATA_SYSTEM_OS_DETECTION=%s"
+                    "&NETDATA_SYSTEM_KERNEL_NAME=%s"
+                    "&NETDATA_SYSTEM_KERNEL_VERSION=%s"
+                    "&NETDATA_SYSTEM_ARCHITECTURE=%s"
+                    "&NETDATA_SYSTEM_VIRTUALIZATION=%s"
+                    "&NETDATA_SYSTEM_VIRT_DETECTION=%s"
+                    "&NETDATA_SYSTEM_CONTAINER=%s"
+                    "&NETDATA_SYSTEM_CONTAINER_DETECTION=%s"
+                    " HTTP/1.1\r\n"
                     "User-Agent: %s/%s\r\n"
                     "Accept: */*\r\n\r\n"
               , host->rrdpush_send_api_key
@@ -456,6 +470,19 @@ static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_po
               , host->os
               , host->timezone
               , (host->tags)?host->tags:""
+              , host->system_info->os_name
+              , host->system_info->os_id
+              , host->system_info->os_id_like
+              , host->system_info->os_version
+              , host->system_info->os_version_id
+              , host->system_info->os_detection
+              , host->system_info->kernel_name
+              , host->system_info->kernel_version
+              , host->system_info->architecture
+              , host->system_info->virtualization
+              , host->system_info->virt_detection
+              , host->system_info->container
+              , host->system_info->container_detection
               , host->program_name
               , host->program_version
     );
@@ -821,6 +848,7 @@ static int rrdpush_receive(int fd
                            , const char *tags
                            , const char *program_name
                            , const char *program_version
+                           , struct rrdhost_system_info *system_info
                            , int update_every
                            , char *client_ip
                            , char *client_port
@@ -894,6 +922,7 @@ static int rrdpush_receive(int fd
                 , rrdpush_destination
                 , rrdpush_api_key
                 , rrdpush_send_charts_matching
+                , system_info
         );
 
     if(!host) {
@@ -1027,6 +1056,7 @@ struct rrdpush_thread {
     char *client_port;
     char *program_name;
     char *program_version;
+    struct rrdhost_system_info *system_info;
     int update_every;
 };
 
@@ -1049,6 +1079,7 @@ static void rrdpush_receiver_thread_cleanup(void *ptr) {
         freez(rpt->client_port);
         freez(rpt->program_name);
         freez(rpt->program_version);
+        rrdhost_system_info_free(rpt->system_info);
         freez(rpt);
     }
 }
@@ -1070,6 +1101,7 @@ static void *rrdpush_receiver_thread(void *ptr) {
 	    , rpt->tags
 	    , rpt->program_name
 	    , rpt->program_version
+        , rpt->system_info
 	    , rpt->update_every
 	    , rpt->client_ip
 	    , rpt->client_port
@@ -1120,6 +1152,8 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
     int update_every = default_rrd_update_every;
     char buf[GUID_LEN + 1];
 
+    struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));
+
     while(url) {
         char *value = mystrsep(&url, "&");
         if(!value || !*value) continue;
@@ -1145,7 +1179,9 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
         else if(!strcmp(name, "tags"))
             tags = value;
         else
-            info("STREAM [receive from [%s]:%s]: request has parameter '%s' = '%s', which is not used.", w->client_ip, w->client_port, key, value);
+            if(unlikely(rrdhost_set_system_info_variable(system_info, name, value))) {
+                info("STREAM [receive from [%s]:%s]: request has parameter '%s' = '%s', which is not used.", w->client_ip, w->client_port, key, value);
+            }
     }
 
     if(!key || !*key) {
@@ -1248,6 +1284,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
     rpt->client_ip         = strdupz(w->client_ip);
     rpt->client_port       = strdupz(w->client_port);
     rpt->update_every      = update_every;
+    rpt->system_info       = system_info;
 
     if(w->user_agent && w->user_agent[0]) {
         char *t = strchr(w->user_agent, '/');
@@ -1260,6 +1297,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
         if(t && *t) rpt->program_version = strdupz(t);
     }
 
+
     netdata_thread_t thread;
 
     debug(D_SYSTEM, "starting STREAM receive thread.");
diff --git a/web/api/web_api_v1.c b/web/api/web_api_v1.c
index 0499dd6718..04c9198cca 100644
--- a/web/api/web_api_v1.c
+++ b/web/api/web_api_v1.c
@@ -736,20 +736,20 @@ inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, c
     web_client_api_request_v1_info_summary_alarm_statuses(host, wb);
     buffer_strcat(wb, "\t},\n");
 
-    buffer_sprintf(wb, "\t\"os_name\": %s,\n", getenv("NETDATA_SYSTEM_OS_NAME"));
-    buffer_sprintf(wb, "\t\"os_id\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_ID"));
-    buffer_sprintf(wb, "\t\"os_id_like\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_ID_LIKE"));
-    buffer_sprintf(wb, "\t\"os_version\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_VERSION"));
-    buffer_sprintf(wb, "\t\"os_version_id\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_VERSION_ID"));
-    buffer_sprintf(wb, "\t\"os_detection\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_DETECTION"));
-    buffer_sprintf(wb, "\t\"kernel_name\": \"%s\",\n", getenv("NETDATA_SYSTEM_KERNEL_NAME"));
-    buffer_sprintf(wb, "\t\"kernel_version\": \"%s\",\n", getenv("NETDATA_SYSTEM_KERNEL_VERSION"));
-    buffer_sprintf(wb, "\t\"architecture\": \"%s\",\n", getenv("NETDATA_SYSTEM_ARCHITECTURE"));
-    buffer_sprintf(wb, "\t\"virtualization\": \"%s\",\n", getenv("NETDATA_SYSTEM_VIRTUALIZATION"));
-    buffer_sprintf(wb, "\t\"virt_detection\": \"%s\",\n", getenv("NETDATA_SYSTEM_VIRT_DETECTION"));
-    buffer_sprintf(wb, "\t\"container\": \"%s\",\n", getenv("NETDATA_SYSTEM_CONTAINER"));
-    buffer_sprintf(wb, "\t\"container_detection\": \"%s\",\n", getenv("NETDATA_SYSTEM_CONTAINER_DETECTION"));
-    
+    buffer_sprintf(wb, "\t\"os_name\": %s,\n", host->system_info->os_name);
+    buffer_sprintf(wb, "\t\"os_id\": \"%s\",\n", host->system_info->os_id);
+    buffer_sprintf(wb, "\t\"os_id_like\": \"%s\",\n", host->system_info->os_id_like);
+    buffer_sprintf(wb, "\t\"os_version\": \"%s\",\n", host->system_info->os_version);
+    buffer_sprintf(wb, "\t\"os_version_id\": \"%s\",\n", host->system_info->os_version_id);
+    buffer_sprintf(wb, "\t\"os_detection\": \"%s\",\n", host->system_info->os_detection);
+    buffer_sprintf(wb, "\t\"kernel_name\": \"%s\",\n", host->system_info->kernel_name);
+    buffer_sprintf(wb, "\t\"kernel_version\": \"%s\",\n", host->system_info->kernel_version);
+    buffer_sprintf(wb, "\t\"architecture\": \"%s\",\n", host->system_info->architecture);
+    buffer_sprintf(wb, "\t\"virtualization\": \"%s\",\n", host->system_info->virtualization);
+    buffer_sprintf(wb, "\t\"virt_detection\": \"%s\",\n", host->system_info->virt_detection);
+    buffer_sprintf(wb, "\t\"container\": \"%s\",\n", host->system_info->container);
+    buffer_sprintf(wb, "\t\"container_detection\": \"%s\",\n", host->system_info->container_detection);
+
     buffer_strcat(wb, "\t\"collectors\": [");
     chartcollectors2json(host, wb);
     buffer_strcat(wb, "\n\t]\n");