Support SOCKS5 in ACLK Challenge/Response and rewrite with LWS (#8404)

* wip

* add alpn

* beggining of cleanup

* move common code

* add SOCKS5 support

* check HTTP response code

* add timeout

* separate https_client into own files

* fix some mem leaks from master + avoid string copying and alloc/free

* fix some PR unrelated warnings

8 files changed, 379 insertions, 183 deletions

diff --git a/aclk/aclk_common.c b/aclk/aclk_common.c
index 33c7c2e0ea..fe113478f5 100644
--- a/aclk/aclk_common.c
+++ b/aclk/aclk_common.c
@@ -1,5 +1,7 @@
 #include "aclk_common.h"
 
+#include "../daemon/common.h"
+
 struct {
     ACLK_PROXY_TYPE type;
     const char *url_str;
@@ -33,3 +35,73 @@ ACLK_PROXY_TYPE aclk_verify_proxy(const char *string)
 
     return aclk_find_proxy(string);
 }
+
+// helper function to censor user&password
+// for logging purposes
+void safe_log_proxy_censor(char *proxy) {
+    size_t length = strlen(proxy);
+    char *auth = proxy+length-1;
+    char *cur;
+
+    while( (auth >= proxy) && (*auth != '@') )
+        auth--;
+
+    //if not found or @ is first char do nothing
+    if(auth<=proxy)
+        return;
+
+    cur = strstr(proxy, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
+    if(!cur)
+        cur = proxy;
+    else
+        cur += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
+
+    while(cur < auth) {
+        *cur='X';
+        cur++;
+    }
+}
+
+static inline void safe_log_proxy_error(char *str, const char *proxy) {
+    char *log = strdupz(proxy);
+    safe_log_proxy_censor(log);
+    error("%s Provided Value:\"%s\"", str, log);
+    freez(log);
+}
+
+static inline int check_socks_enviroment(const char **proxy) {
+    char *tmp = getenv("socks_proxy");
+
+    if(!tmp)
+        return 1;
+
+    if(aclk_verify_proxy(tmp) == PROXY_TYPE_SOCKS5) {
+        *proxy = tmp;
+        return 0;
+    }
+
+    safe_log_proxy_error("Environment var \"socks_proxy\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".", tmp);
+    return 1;
+}
+
+const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type) {
+    const char *proxy = config_get(CONFIG_SECTION_ACLK, ACLK_PROXY_CONFIG_VAR, ACLK_PROXY_ENV);
+    *type = PROXY_DISABLED;
+
+    if(strcmp(proxy, "none") == 0)
+        return proxy;
+
+    if(strcmp(proxy, ACLK_PROXY_ENV) == 0) {
+        if(check_socks_enviroment(&proxy) == 0)
+            *type = PROXY_TYPE_SOCKS5;
+        return proxy;
+    }
+
+    *type = aclk_verify_proxy(proxy);
+    if(*type == PROXY_TYPE_UNKNOWN) {
+        *type = PROXY_DISABLED;
+        safe_log_proxy_error("Config var \"" ACLK_PROXY_CONFIG_VAR "\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".", proxy);
+    }
+
+    return proxy;
+}
diff --git a/aclk/aclk_common.h b/aclk/aclk_common.h
index a38391a761..4b2d6c76c8 100644
--- a/aclk/aclk_common.h
+++ b/aclk/aclk_common.h
@@ -16,5 +16,8 @@ typedef enum aclk_proxy_type {
 #define ACLK_PROXY_CONFIG_VAR "proxy"
 
 ACLK_PROXY_TYPE aclk_verify_proxy(const char *string);
+const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type);
+void safe_log_proxy_censor(char *proxy);
+int aclk_decode_base_url(char *url, char **aclk_hostname, char **aclk_port);
 
 #endif //ACLK_COMMON_H
diff --git a/aclk/aclk_lws_https_client.c b/aclk/aclk_lws_https_client.c
new file mode 100644
index 0000000000..c07e185baa
--- /dev/null
+++ b/aclk/aclk_lws_https_client.c
@@ -0,0 +1,246 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#define ACLK_LWS_HTTPS_CLIENT_INTERNAL
+#include "aclk_lws_https_client.h"
+
+#include "aclk_common.h"
+
+#include "aclk_lws_wss_client.h"
+
+#define SMALL_BUFFER 16
+
+struct simple_hcc_data {
+    char *data;
+    size_t data_size;
+    char *payload;
+    int response_code;
+    int done;
+};
+
+static int simple_https_client_callback(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len)
+{
+    int n;
+    char *ptr;
+    char buffer[SMALL_BUFFER];
+    struct simple_hcc_data *perconn_data = lws_get_opaque_user_data(wsi);
+
+    switch (reason) {
+    case LWS_CALLBACK_RECEIVE_CLIENT_HTTP_READ:
+        debug(D_ACLK, "LWS_CALLBACK_RECEIVE_CLIENT_HTTP_READ");
+        return 0;
+    case LWS_CALLBACK_RECEIVE_CLIENT_HTTP:
+        debug(D_ACLK, "LWS_CALLBACK_RECEIVE_CLIENT_HTTP");
+        if(!perconn_data) {
+            error("Missing Per Connect Data");
+            return -1;
+        }
+        ptr = perconn_data->data;
+        n = perconn_data->data_size;
+        if (lws_http_client_read(wsi, &ptr, &n) < 0)
+            return -1;
+        return 0;
+    case LWS_CALLBACK_WSI_DESTROY:
+        debug(D_ACLK, "LWS_CALLBACK_WSI_DESTROY");
+        if(perconn_data)
+            perconn_data->done = 1;
+        return 0;
+    case LWS_CALLBACK_ESTABLISHED_CLIENT_HTTP:
+        debug(D_ACLK, "LWS_CALLBACK_ESTABLISHED_CLIENT_HTTP");
+        if(perconn_data)
+            perconn_data->response_code = lws_http_client_http_response(wsi);
+        return 0;
+    case LWS_CALLBACK_CLOSED_CLIENT_HTTP:
+        debug(D_ACLK, "LWS_CALLBACK_CLOSED_CLIENT_HTTP");
+        return 0;
+    case LWS_CALLBACK_OPENSSL_LOAD_EXTRA_CLIENT_VERIFY_CERTS:
+        debug(D_ACLK, "LWS_CALLBACK_OPENSSL_LOAD_EXTRA_CLIENT_VERIFY_CERTS");
+        return 0;
+    case LWS_CALLBACK_CLIENT_APPEND_HANDSHAKE_HEADER:
+        debug(D_ACLK, "LWS_CALLBACK_CLIENT_APPEND_HANDSHAKE_HEADER");
+        if(perconn_data && perconn_data->payload) {
+            unsigned char **p = (unsigned char **)in, *end = (*p) + len;
+            snprintfz(buffer, SMALL_BUFFER, "%zu", strlen(perconn_data->payload));
+            if (lws_add_http_header_by_token(wsi,
+                    WSI_TOKEN_HTTP_CONTENT_LENGTH,
+                    (unsigned char *)buffer, strlen(buffer), p, end))
+                return -1;
+            if (lws_add_http_header_by_token(wsi,
+                    WSI_TOKEN_HTTP_CONTENT_TYPE,
+                    (unsigned char *)ACLK_CONTENT_TYPE_JSON,
+                    strlen(ACLK_CONTENT_TYPE_JSON), p, end))
+                return -1;
+            lws_client_http_body_pending(wsi, 1);
+            lws_callback_on_writable(wsi);
+        }
+        return 0;
+    case LWS_CALLBACK_CLIENT_HTTP_WRITEABLE:
+        debug(D_ACLK, "LWS_CALLBACK_CLIENT_HTTP_WRITEABLE");
+        if(perconn_data && perconn_data->payload) {
+            n = strlen(perconn_data->payload);
+            if(perconn_data->data_size < LWS_PRE + n + 1) {
+                error("Buffer given is not big enough");
+                return 1;
+            }
+
+            memcpy(&perconn_data->data[LWS_PRE], perconn_data->payload, n);
+            if(n != lws_write(wsi, (unsigned char*)&perconn_data->data[LWS_PRE], n, LWS_WRITE_HTTP)) {
+                error("lws_write error");
+                perconn_data->data[0] = 0;
+                return 1;
+            }
+            lws_client_http_body_pending(wsi, 0);
+            // clean for subsequent reply read
+            perconn_data->data[0] = 0;
+        }
+        return 0;
+    case LWS_CALLBACK_CLIENT_HTTP_BIND_PROTOCOL:
+        debug(D_ACLK, "LWS_CALLBACK_CLIENT_HTTP_BIND_PROTOCOL");
+        return 0;
+    case LWS_CALLBACK_WSI_CREATE:
+        debug(D_ACLK, "LWS_CALLBACK_WSI_CREATE");
+        return 0;
+    case LWS_CALLBACK_PROTOCOL_INIT:
+        debug(D_ACLK, "LWS_CALLBACK_PROTOCOL_INIT");
+        return 0;
+    case LWS_CALLBACK_CLIENT_HTTP_DROP_PROTOCOL:
+        debug(D_ACLK, "LWS_CALLBACK_CLIENT_HTTP_DROP_PROTOCOL");
+        return 0;
+    case LWS_CALLBACK_SERVER_NEW_CLIENT_INSTANTIATED:
+        debug(D_ACLK, "LWS_CALLBACK_SERVER_NEW_CLIENT_INSTANTIATED");
+        return 0;
+    case LWS_CALLBACK_GET_THREAD_ID:
+        debug(D_ACLK, "LWS_CALLBACK_GET_THREAD_ID");
+        return 0;
+    case LWS_CALLBACK_EVENT_WAIT_CANCELLED:
+        debug(D_ACLK, "LWS_CALLBACK_EVENT_WAIT_CANCELLED");
+        return 0;
+    case LWS_CALLBACK_OPENSSL_PERFORM_SERVER_CERT_VERIFICATION:
+        debug(D_ACLK, "LWS_CALLBACK_OPENSSL_PERFORM_SERVER_CERT_VERIFICATION");
+        return 0;
+    case LWS_CALLBACK_CLIENT_FILTER_PRE_ESTABLISH:
+        debug(D_ACLK, "LWS_CALLBACK_CLIENT_FILTER_PRE_ESTABLISH");
+        return 0;
+    default:
+        debug(D_ACLK, "Unknown callback %d", (int)reason);
+        return 0;
+    }
+}
+
+static const struct lws_protocols protocols[] = {
+    {
+        "http",
+        simple_https_client_callback,
+        0,
+        0,
+    },
+    { NULL, NULL, 0, 0 }
+};
+
+static void simple_hcc_log_divert(int level, const char *line)
+{
+    error("Libwebsockets: %s", line);
+}
+
+int aclk_send_https_request(char *method, char *host, char *port, char *url, char *b, size_t b_size, char *payload)
+{
+    info("%s %s", __func__, method);
+
+    struct lws_context_creation_info info;
+    struct lws_client_connect_info i;
+    struct lws_context *context;
+
+    struct simple_hcc_data *data = callocz(1, sizeof(struct simple_hcc_data));
+    data->data = b;
+    data->data[0] = 0;
+    data->data_size = b_size;
+    data->payload = payload;
+
+    int n = 0;
+    time_t timestamp;
+
+    //TODO -> deduplicate (aclk_lws_wss_connect)
+    static const char *proxy = NULL;
+    static ACLK_PROXY_TYPE proxy_type = PROXY_NOT_SET;
+    struct lws_vhost *vhost;
+    char *log;
+
+    if(proxy_type == PROXY_NOT_SET)
+        proxy = aclk_lws_wss_get_proxy_setting(&proxy_type);
+
+
+    memset(&info, 0, sizeof info);
+
+    info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
+    info.port = CONTEXT_PORT_NO_LISTEN;
+    info.protocols = protocols;
+
+
+    context = lws_create_context(&info);
+    if (!context) {
+        error("Error creating LWS context");
+        return 1;
+    }
+
+    lws_set_log_level(LLL_ERR | LLL_WARN, simple_hcc_log_divert);
+
+    lws_service(context, 0);
+
+    memset(&i, 0, sizeof i); /* otherwise uninitialized garbage */
+    i.context = context;
+
+#ifdef ACLK_SSL_ALLOW_SELF_SIGNED
+    i.ssl_connection = LCCSCF_USE_SSL | LCCSCF_ALLOW_SELFSIGNED | LCCSCF_SKIP_SERVER_CERT_HOSTNAME_CHECK;
+    info("Disabling SSL certificate checks");
+#else
+    i.ssl_connection = LCCSCF_USE_SSL;
+#endif
+
+    i.port = atoi(port);
+    i.address = host;
+    i.path = url;
+
+    i.host = i.address;
+    i.origin = i.address;
+    i.method = method;
+    i.opaque_user_data = data;
+    i.alpn = "http/1.1";
+
+    i.protocol = protocols[0].name;
+
+    vhost = lws_get_vhost_by_name(context, "default");
+    if(!vhost)
+        fatal("Could not find the default LWS vhost.");
+
+    lws_set_socks(vhost, ":");
+    lws_set_proxy(vhost, ":");
+
+    if(proxy_type == PROXY_TYPE_SOCKS5) {
+        log = strdupz(proxy);
+        safe_log_proxy_censor(log);
+        info("Connecting using SOCKS5 proxy:\"%s\"", log);
+        freez(log);
+        if(aclk_wss_set_socks(vhost, proxy))
+            error("LWS failed to accept socks proxy.");
+    }
+
+    lws_client_connect_via_info(&i);
+
+    // libwebsockets handle connection timeouts already
+    // this adds additional safety in case of bug in LWS
+    timestamp = now_monotonic_sec();
+    while( n >= 0 && !data->done && !netdata_exit) {
+        n = lws_service(context, 0);
+        if( now_monotonic_sec() - timestamp > SEND_HTTPS_REQUEST_TIMEOUT ) {
+            data->data[0] = 0;
+            data->done = 1;
+            error("Servicing LWS took too long.");
+        }
+    }
+
+    lws_context_destroy(context);
+
+    n = data->response_code;
+
+    freez(data);
+    return (n < 200 || n >= 300);
+}
diff --git a/aclk/aclk_lws_https_client.h b/aclk/aclk_lws_https_client.h
new file mode 100644
index 0000000000..7a87850cef
--- /dev/null
+++ b/aclk/aclk_lws_https_client.h
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_LWS_HTTPS_CLIENT_H
+#define NETDATA_LWS_HTTPS_CLIENT_H
+
+#include "../daemon/common.h"
+#include "libnetdata/libnetdata.h"
+
+#define DATAMAXLEN 1024*16
+
+#ifdef ACLK_LWS_HTTPS_CLIENT_INTERNAL
+#define ACLK_CONTENT_TYPE_JSON "application/json"
+#define SEND_HTTPS_REQUEST_TIMEOUT 30
+#endif
+
+int aclk_send_https_request(char *method, char *host, char *port, char *url, char *b, size_t b_size, char *payload);
+
+#endif /* NETDATA_LWS_HTTPS_CLIENT_H */
diff --git a/aclk/aclk_lws_wss_client.c b/aclk/aclk_lws_wss_client.c
index f46b1f0bce..a4198c94f8 100644
--- a/aclk/aclk_lws_wss_client.c
+++ b/aclk/aclk_lws_wss_client.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "aclk_lws_wss_client.h"
 
 #include "libnetdata/libnetdata.h"
@@ -188,7 +190,7 @@ failure_cleanup_2:
     return 1;
 }
 
-void aclk_lws_wss_client_destroy(struct aclk_lws_wss_engine_instance *engine_instance)
+void aclk_lws_wss_client_destroy()
 {
     if (engine_instance == NULL)
         return;
@@ -204,89 +206,18 @@ void aclk_lws_wss_client_destroy(struct aclk_lws_wss_engine_instance *engine_ins
 #endif
 }
 
-static int _aclk_wss_set_socks(struct lws_vhost *vhost, const char *socks)
-{
-	char *proxy = strstr(socks, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
-
-	if(!proxy)
-		return -1;
-
-	proxy += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
-
-	if(!*proxy)
-		return -1;
-
-	return lws_set_socks(vhost, proxy);
-}
-
-// helper function to censor user&password
-// for logging purposes
-static void safe_log_proxy_censor(char *proxy) {
-    size_t length = strlen(proxy);
-    char *auth = proxy+length-1;
-    char *cur;
+int aclk_wss_set_socks(struct lws_vhost *vhost, const char *socks) {
+    char *proxy = strstr(socks, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
 
-    while( (auth >= proxy) && (*auth != '@') )
-        auth--;
+    if(!proxy)
+        return -1;
 
-    //if not found or @ is first char do nothing
-    if(auth<=proxy)
-        return;
-
-    cur = strstr(proxy, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
-    if(!cur)
-        cur = proxy;
-    else
-        cur += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
-
-    while(cur < auth) {
-        *cur='X';
-        cur++;
-    }
-}
-
-static inline void safe_log_proxy_error(char *str, const char *proxy) {
-    char *log = strdupz(proxy);
-    safe_log_proxy_censor(log);
-    error("%s Provided Value:\"%s\"", str, log);
-    freez(log);
-}
+    proxy += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
 
-static inline int check_socks_enviroment(const char **proxy) {
-    char *tmp = getenv("socks_proxy");
-
-    if(!tmp)
-        return 1;
-
-    if(aclk_verify_proxy(tmp) == PROXY_TYPE_SOCKS5) {
-        *proxy = tmp;
-        return 0;
-    }
-
-    safe_log_proxy_error("Environment var \"socks_proxy\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".", tmp);
-    return 1;
-}
-
-static const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type) {
-    const char *proxy = config_get(CONFIG_SECTION_ACLK, ACLK_PROXY_CONFIG_VAR, ACLK_PROXY_ENV);
-    *type = PROXY_DISABLED;
-
-    if(strcmp(proxy, "none") == 0)
-        return proxy;
-
-    if(strcmp(proxy, ACLK_PROXY_ENV) == 0) {
-        if(check_socks_enviroment(&proxy) == 0)
-            *type = PROXY_TYPE_SOCKS5;
-        return proxy;
-    }
-
-    *type = aclk_verify_proxy(proxy);
-    if(*type == PROXY_TYPE_UNKNOWN) {
-        *type = PROXY_DISABLED;
-        safe_log_proxy_error("Config var \"" ACLK_PROXY_CONFIG_VAR "\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".", proxy);
-    }
+    if(!*proxy)
+        return -1;
 
-    return proxy;
+    return lws_set_socks(vhost, proxy);
 }
 
 // Return code indicates if connection attempt has started async.
@@ -338,7 +269,7 @@ int aclk_lws_wss_connect(char *host, int port)
         safe_log_proxy_censor(log);
         info("Connecting using SOCKS5 proxy:\"%s\"", log);
         freez(log);
-        if(_aclk_wss_set_socks(vhost, proxy))
+        if(aclk_wss_set_socks(vhost, proxy))
             error("LWS failed to accept socks proxy.");
         break;
     default:
diff --git a/aclk/aclk_lws_wss_client.h b/aclk/aclk_lws_wss_client.h
index c6a7e10120..42a0ab4dfe 100644
--- a/aclk/aclk_lws_wss_client.h
+++ b/aclk/aclk_lws_wss_client.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #ifndef ACLK_LWS_WSS_CLIENT_H
 #define ACLK_LWS_WSS_CLIENT_H
 
@@ -79,5 +81,7 @@ void aclk_lws_connection_data_received();
 void aclk_lws_connection_closed();
 void lws_wss_check_queues(size_t *write_len, size_t *write_len_bytes, size_t *read_len);
 
+int aclk_wss_set_socks(struct lws_vhost *vhost, const char *socks);
+
 #define FRAGMENT_SIZE 4096
 #endif
diff --git a/aclk/agent_cloud_link.c b/aclk/agent_cloud_link.c
index 186ff68a93..ab1b85bae1 100644
--- a/aclk/agent_cloud_link.c
+++ b/aclk/agent_cloud_link.c
@@ -2,6 +2,7 @@
 
 #include "libnetdata/libnetdata.h"
 #include "agent_cloud_link.h"
+#include "aclk_lws_https_client.h"
 
 // State-machine for the on-connect metadata transmission.
 // TODO: The AGENT_STATE should be centralized as it would be useful to control error-logging during the initial
@@ -176,7 +177,6 @@ static int create_private_key()
     char err[512];
     ERR_error_string_n(ERR_get_error(), err, sizeof(err));
     error("Claimed agent cannot establish ACLK - cannot create private key: %s", err);
-    freez(err);
 
 biofailed:
     freez(private_key);
@@ -946,75 +946,6 @@ static void aclk_main_cleanup(void *ptr)
     static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
 }
 
-int send_https_request(char *method, char *host, char *port, char *url, BUFFER *b, char *payload)
-{
-    struct timeval timeout = { .tv_sec = 30, .tv_usec = 0 };
-    int rc=1;
-
-    size_t payload_len = 0;
-    if (payload != NULL)
-        payload_len = strlen(payload);
-
-    buffer_flush(b);
-    buffer_sprintf(
-        b,
-        "%s %s HTTP/1.1\r\nHost: %s\r\nAccept: plain/text\r\nContent-length: %zu\r\nAccept-Language: en-us\r\n"
-        "User-Agent: Netdata/rocks\r\n\r\n",
-        method, url, host, payload_len);
-    if (payload != NULL)
-        buffer_strcat(b, payload);
-    debug(D_ACLK, "Sending HTTPS req (%zu bytes): '%s'", b->len, buffer_tostring(b));
-    int sock = connect_to_this_ip46(IPPROTO_TCP, SOCK_STREAM, host, 0, port, &timeout);
-
-    if (unlikely(sock == -1)) {
-        error("Handshake failed");
-        return 1;
-    }
-
-    SSL_CTX *ctx = security_initialize_openssl_client();
-    if (ctx==NULL) {
-        error("Cannot allocate SSL context");
-        goto exit_sock;
-    }
-    // Certificate chain: not updating the stores - do we need private CA roots?
-    // Calls to SSL_CTX_load_verify_locations would go here.
-    SSL *ssl = SSL_new(ctx);
-    if (ssl==NULL) {
-        error("Cannot allocate SSL");
-        goto exit_CTX;
-    }
-    SSL_set_fd(ssl, sock);
-    int err = SSL_connect(ssl);
-    if (err!=1) {
-        error("SSL_connect() failed with err=%d", err);
-        goto exit_SSL;
-    }
-    err = SSL_write(ssl, b->buffer, b->len);
-    if (err <= 0)
-    {
-        error("SSL_write() failed with err=%d", err);
-        goto exit_SSL;
-    }
-    buffer_flush(b);
-    int bytes_read = SSL_read(ssl, b->buffer, b->size);
-    if (bytes_read >= 0) {
-        debug(D_ACLK, "Received %d bytes in response", bytes_read);
-        b->len = bytes_read;
-    }
-    else {
-        error("No response available - SSL_read()=%d", bytes_read);
-    }
-    SSL_shutdown(ssl);
-    rc = 0;
-exit_SSL:
-    SSL_free(ssl);
-exit_CTX:
-    SSL_CTX_free(ctx);
-exit_sock:
-    close(sock);
-    return rc;
-}
-
 struct dictionary_singleton {
     char *key;
     char *result;
@@ -1242,6 +1173,7 @@ int host_end = pos;
 
 void aclk_get_challenge(char *aclk_hostname, char *aclk_port)
 {
+    char *data_buffer = mallocz(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
     debug(D_ACLK, "Performing challenge-response sequence");
     if (aclk_password != NULL)
     {
@@ -1249,44 +1181,36 @@ void aclk_get_challenge(char *aclk_hostname, char *aclk_port)
         aclk_password = NULL;
     }
     // curl http://cloud-iam-agent-service:8080/api/v1/auth/node/00000000-0000-0000-0000-000000000000/challenge
-    BUFFER *b = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
     // TODO - target host?
     char *agent_id = is_agent_claimed();
     if (agent_id == NULL)
     {
         error("Agent was not claimed - cannot perform challenge/response");
-        return;
+        goto CLEANUP;
     }
     char url[1024];
     sprintf(url, "/api/v1/auth/node/%s/challenge", agent_id);
     info("Retrieving challenge from cloud: %s %s %s", aclk_hostname, aclk_port, url);
-    if(send_https_request("GET", aclk_hostname, aclk_port, url, b, NULL))
+    if(aclk_send_https_request("GET", aclk_hostname, aclk_port, url, data_buffer, NETDATA_WEB_RESPONSE_INITIAL_SIZE, NULL))
     {
         error("Challenge failed");
-        return;
+        goto CLEANUP;
     }
     struct dictionary_singleton challenge = { .key = "challenge", .result = NULL };
-    // Force null-termination?
-    char *payload = NULL;
-    payload = extract_payload(b);
-    if (payload==NULL) {
-      error("Could not find payload in http response #1 (the challenge):\n%s", b->buffer);
-      return;
-    }
-    debug(D_ACLK, "Challenge response from cloud: %s", payload);
-    if (json_parse(payload, &challenge, json_extract_singleton) != JSON_OK)
+
+    debug(D_ACLK, "Challenge response from cloud: %s", data_buffer);
+    if ( json_parse(data_buffer, &challenge, json_extract_singleton) != JSON_OK)
     {
         freez(challenge.result);
-        error("Could not parse the json response with the challenge: %s", payload);
-        return;
+        error("Could not parse the json response with the challenge: %s", data_buffer);
+        goto CLEANUP;
     }
     if (challenge.result == NULL ) {
-        error("Could not retrieve challenge from auth response: %s", payload);
-        return;
+        error("Could not retrieve challenge from auth response: %s", data_buffer);
+        goto CLEANUP;
     }
 
 
-
     size_t challenge_len = strlen(challenge.result);
     unsigned char decoded[512];
     size_t decoded_len = base64_decode((unsigned char*)challenge.result, challenge_len, decoded, sizeof(decoded));
@@ -1304,29 +1228,25 @@ void aclk_get_challenge(char *aclk_hostname, char *aclk_port)
     debug(D_ACLK, "Password phase: %s",response_json);
     // TODO - host
     sprintf(url, "/api/v1/auth/node/%s/password", agent_id);
-    if(send_https_request("POST", aclk_hostname, aclk_port, url, b, response_json))
+    if(aclk_send_https_request("POST", aclk_hostname, aclk_port, url, data_buffer, NETDATA_WEB_RESPONSE_INITIAL_SIZE, response_json))
     {
         error("Challenge-response failed");
-        return;
-    }
-    payload = extract_payload(b);
-    if (payload==NULL) {
-      error("Could not find payload in http response #2 (the password):\n%s", b->buffer);
-      return;
+        goto CLEANUP;
     }
-    debug(D_ACLK, "Password response from cloud: %s", payload);
+
+    debug(D_ACLK, "Password response from cloud: %s", data_buffer);
 
     struct dictionary_singleton password = { .key = "password", .result = NULL };
-    if (json_parse(payload, &password, json_extract_singleton) != JSON_OK)
+    if ( json_parse(data_buffer, &password, json_extract_singleton) != JSON_OK)
     {
         freez(password.result);
-        error("Could not parse the json response with the password: %s", payload);
-        return;
+        error("Could not parse the json response with the password: %s", data_buffer);
+        goto CLEANUP;
     }
 
     if (password.result == NULL ) {
         error("Could not retrieve password from auth response");
-        return;
+        goto CLEANUP;
     }
     if (aclk_password != NULL )
         freez(aclk_password);
@@ -1334,7 +1254,9 @@ void aclk_get_challenge(char *aclk_hostname, char *aclk_port)
         aclk_username = strdupz(agent_id);
     aclk_password = password.result;
 
-    buffer_free(b);
+CLEANUP:
+    freez(data_buffer);
+    return;
 }
 
 static void aclk_try_to_connect(char *hostname, char *port, int port_num)
diff --git a/aclk/mqtt.c b/aclk/mqtt.c
index 59ed46e362..2010d9a326 100644
--- a/aclk/mqtt.c
+++ b/aclk/mqtt.c
@@ -284,7 +284,7 @@ int _link_send_message(char *topic, unsigned char *message, int *mid)
     if (unlikely(rc != MOSQ_ERR_SUCCESS))
         return rc;
 
-    int msg_len = strlen(message);
+    int msg_len = strlen((char*)message);
     error("Sending MQTT len=%d starts %02x %02x %02x", msg_len, message[0], message[1], message[2]);
     rc = mosquitto_publish(mosq, mid, topic, msg_len, message, ACLK_QOS, 0);