Add http headers to responses (#8760)

The MQTT payloads for responses to API requests from the cloud now include a headers field with the raw http headers encoded into unicode. This exposes the `Date` and `Expired` fields to the cloud backend.

7 files changed, 73 insertions, 53 deletions

diff --git a/aclk/agent_cloud_link.c b/aclk/agent_cloud_link.c
index e59c9ea413..d3bf881a9c 100644
--- a/aclk/agent_cloud_link.c
+++ b/aclk/agent_cloud_link.c
@@ -723,6 +723,50 @@ void aclk_del_collector(const char *hostname, const char *plugin_name, const cha
     }
 
     _free_collector(tmp_collector);
+
+}
+/*
+ * Take a buffer, encode it and rewrite it
+ *
+ */
+
+static char *aclk_encode_response(char *src, size_t content_size, int keep_newlines)
+{
+    char *tmp_buffer = mallocz(content_size * 2);
+    char *dst = tmp_buffer;
+    while (content_size > 0) {
+        switch (*src) {
+            case '\n':
+                if (keep_newlines)
+                {
+                    *dst++ = '\\';
+                    *dst++ = 'n';
+                }
+                break;
+            case '\t':
+                break;
+            case 0x01 ... 0x08:
+            case 0x0b ... 0x1F:
+                *dst++ = '\\';
+                *dst++ = 'u';
+                *dst++ = '0';
+                *dst++ = '0';
+                *dst++ = (*src < 0x0F) ? '0' : '1';
+                *dst++ = to_hex(*src);
+                break;
+            case '\"':
+                *dst++ = '\\';
+                *dst++ = *src;
+                break;
+            default:
+                *dst++ = *src;
+        }
+        src++;
+        content_size--;
+    }
+    *dst = '\0';
+
+    return tmp_buffer;
 }
 
 int aclk_execute_query(struct aclk_query *this_query)
@@ -730,6 +774,8 @@ int aclk_execute_query(struct aclk_query *this_query)
     if (strncmp(this_query->query, "/api/v1/", 8) == 0) {
         struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
         w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
+        w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
+        w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
         strcpy(w->origin, "*"); // Simulate web_client_create_on_fd()
         w->cookie1[0] = 0;      // Simulate web_client_create_on_fd()
         w->cookie2[0] = 0;      // Simulate web_client_create_on_fd()
@@ -745,26 +791,36 @@ int aclk_execute_query(struct aclk_query *this_query)
         mysep = strrchr(this_query->query, '/');
 
         // TODO: handle bad response perhaps in a different way. For now it does to the payload
-        int rc = web_client_api_request_v1(localhost, w, mysep ? mysep + 1 : "noop");
+        w->response.code = web_client_api_request_v1(localhost, w, mysep ? mysep + 1 : "noop");
+        now_realtime_timeval(&w->tv_ready);
+        w->response.data->date = w->tv_ready.tv_sec;
+        web_client_build_http_header(w);  // TODO: this function should offset from date, not tv_ready
         BUFFER *local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
         buffer_flush(local_buffer);
         local_buffer->contenttype = CT_APPLICATION_JSON;
 
         aclk_create_header(local_buffer, "http", this_query->msg_id, 0, 0);
         buffer_strcat(local_buffer, ",\n\t\"payload\": ");
-        char *encoded_response = aclk_encode_response(w->response.data);
+        char *encoded_response = aclk_encode_response(w->response.data->buffer, w->response.data->len, 0);
+        char *encoded_header = aclk_encode_response(w->response.header_output->buffer, w->response.header_output->len, 1);
 
         buffer_sprintf(
-            local_buffer, "{\n\"code\": %d,\n\"body\": \"%s\"\n}", rc, encoded_response);
+            local_buffer, "{\n\"code\": %d,\n\"body\": \"%s\",\n\"headers\": \"%s\"\n}", 
+            w->response.code, encoded_response, encoded_header);
 
         buffer_sprintf(local_buffer, "\n}");
 
+        debug(D_ACLK, "Response:%s", encoded_header);
+
         aclk_send_message(this_query->topic, local_buffer->buffer, this_query->msg_id);
 
         buffer_free(w->response.data);
+        buffer_free(w->response.header);
+        buffer_free(w->response.header_output);
         freez(w);
         buffer_free(local_buffer);
         freez(encoded_response);
+        freez(encoded_header);
         return 0;
     }
     return 1;
@@ -1535,46 +1591,6 @@ inline void aclk_create_header(BUFFER *dest, char *type, char *msg_id, time_t ts
     debug(D_ACLK, "Sending v%d msgid [%s] type [%s] time [%ld]", ACLK_VERSION, msg_id, type, ts_secs);
 }
 
-/*
- * Take a buffer, encode it and rewrite it
- *
- */
-
-char *aclk_encode_response(BUFFER *contents)
-{
-    char *tmp_buffer = mallocz(contents->len * 2);
-    char *src, *dst;
-    size_t content_size = contents->len;
-
-    src = contents->buffer;
-    dst = tmp_buffer;
-    while (content_size > 0) {
-        switch (*src) {
-            case '\n':
-            case '\t':
-                break;
-            case 0x01 ... 0x08:
-            case 0x0b ... 0x1F:
-                *dst++ = '\\';
-                *dst++ = '0';
-                *dst++ = '0';
-                *dst++ = (*src < 0x0F) ? '0' : '1';
-                *dst++ = to_hex(*src);
-                break;
-            case '\"':
-                *dst++ = '\\';
-                *dst++ = *src;
-                break;
-            default:
-                *dst++ = *src;
-        }
-        src++;
-        content_size--;
-    }
-    *dst = '\0';
-
-    return tmp_buffer;
-}
 
 /*
  * This will send alarm information which includes
diff --git a/aclk/agent_cloud_link.h b/aclk/agent_cloud_link.h
index f147669e5d..a3722b82ae 100644
--- a/aclk/agent_cloud_link.h
+++ b/aclk/agent_cloud_link.h
@@ -101,7 +101,6 @@ void aclk_del_collector(const char *hostname, const char *plugin_name, const cha
 void aclk_alarm_reload();
 void aclk_send_alarm_metadata();
 int aclk_execute_query(struct aclk_query *query);
-char *aclk_encode_response(BUFFER *contents);
 unsigned long int aclk_reconnect_delay(int mode);
 extern void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host);
 void aclk_single_update_enable();
diff --git a/build_external/bin/clean-install.sh b/build_external/bin/clean-install.sh
index 20bcae66ae..78d1f325fe 100755
--- a/build_external/bin/clean-install.sh
+++ b/build_external/bin/clean-install.sh
@@ -33,19 +33,19 @@ if cat <<HAPPY_CASE | grep "$DISTRO-$VERSION"
 HAPPY_CASE
 then
     docker build -f "$BuildBase/clean-install.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
-            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
+            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" \
             --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
 else
     case "$DISTRO-$VERSION" in
         arch-current)
             docker build -f "$BuildBase/clean-install-arch.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
-            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
-            --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
+            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" \
+            --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED" # --no-cache
         ;;
         arch-extras)   # Add valgrind to the container
             docker build -f "$BuildBase/clean-install-arch-extras.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
-            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
-            --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
+            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" \
+            --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED" # --no-cache
         ;;
         *)
             echo "Unknown $DISTRO-$VERSION"
diff --git a/build_external/clean-install-arch-extras.Dockerfile b/build_external/clean-install-arch-extras.Dockerfile
index 47939039e4..4425283347 100644
--- a/build_external/clean-install-arch-extras.Dockerfile
+++ b/build_external/clean-install-arch-extras.Dockerfile
@@ -22,7 +22,6 @@ RUN pacman --noconfirm --needed -S autoconf \
                                    cmake \
                                    valgrind
 
-ARG ACLK=no
 ARG EXTRA_CFLAGS
 COPY . /opt/netdata/source
 WORKDIR /opt/netdata/source
@@ -46,7 +45,7 @@ RUN rm -rf .git/
 RUN find . -type f >/opt/netdata/manifest
 
 RUN CFLAGS="-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
-    -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --disable-lto
+    -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto
 
 RUN ln -sf /dev/stdout /var/log/netdata/access.log
 RUN ln -sf /dev/stdout /var/log/netdata/debug.log
diff --git a/claim/netdata-claim.sh.in b/claim/netdata-claim.sh.in
index 913ca35c77..50ee660d37 100755
--- a/claim/netdata-claim.sh.in
+++ b/claim/netdata-claim.sh.in
@@ -101,7 +101,7 @@ TOKEN="unknown"
 URL_BASE="https://netdata.cloud"
 ID="unknown"
 ROOMS=""
-HOSTNAME=$(hostname)
+[ -z "$HOSTNAME" ] && HOSTNAME=$(hostname)
 CLOUD_CERTIFICATE_FILE="${CLAIMING_DIR}/cloud_fullchain.pem"
 VERBOSE=0
 INSECURE=0
diff --git a/web/server/web_client.c b/web/server/web_client.c
index 675ac47be2..f6dba80ce2 100644
--- a/web/server/web_client.c
+++ b/web/server/web_client.c
@@ -1118,7 +1118,7 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
     return bytes;
 }
 
-static inline void web_client_send_http_header(struct web_client *w) {
+void web_client_build_http_header(struct web_client *w) {
     if(unlikely(w->response.code != HTTP_RESP_OK))
         buffer_no_cacheable(w->response.data);
 
@@ -1252,6 +1252,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
 
     // end of HTTP header
     buffer_strcat(w->response.header_output, "\r\n");
+}
+
+static inline void web_client_send_http_header(struct web_client *w) {
+    web_client_build_http_header(w);
 
     // sent the HTTP header
     debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %zu: '%s'"
diff --git a/web/server/web_client.h b/web/server/web_client.h
index 3e4b1c3bfa..9e3b851884 100644
--- a/web/server/web_client.h
+++ b/web/server/web_client.h
@@ -207,6 +207,8 @@ extern void buffer_data_options2string(BUFFER *wb, uint32_t options);
 
 extern int mysendfile(struct web_client *w, char *filename);
 
+extern void web_client_build_http_header(struct web_client *w);
+
 #include "daemon/common.h"
 
 #endif