Add HTTP basic authentication to some exporting connectors (#11394)

17 files changed, 128 insertions, 6 deletions

diff --git a/exporting/README.md b/exporting/README.md
index 933de0e075..ef485bb186 100644
--- a/exporting/README.md
+++ b/exporting/README.md
@@ -164,6 +164,8 @@ You can configure each connector individually using the available [options](#opt
 [opentsdb:http:my_opentsdb_http_instance]
     enabled = yes
     destination = localhost:4242
+    username = my_username
+    password = my_password
 
 [opentsdb:https:my_opentsdb_https_instance]
     enabled = yes
diff --git a/exporting/clean_connectors.c b/exporting/clean_connectors.c
index 890e8daac7..4af1219a63 100644
--- a/exporting/clean_connectors.c
+++ b/exporting/clean_connectors.c
@@ -15,6 +15,8 @@ static void clean_instance_config(struct instance_config *config)
     freez((void *)config->type_name);
     freez((void *)config->name);
     freez((void *)config->destination);
+    freez((void *)config->username);
+    freez((void *)config->password);
     freez((void *)config->prefix);
     freez((void *)config->hostname);
 
@@ -49,6 +51,8 @@ void simple_connector_cleanup(struct instance *instance)
     struct simple_connector_data *simple_connector_data =
         (struct simple_connector_data *)instance->connector_specific_data;
 
+    freez(simple_connector_data->auth_string);
+
     buffer_free(instance->buffer);
     buffer_free(simple_connector_data->buffer);
     buffer_free(simple_connector_data->header);
diff --git a/exporting/exporting.conf b/exporting/exporting.conf
index c2e902c051..314e1541ef 100644
--- a/exporting/exporting.conf
+++ b/exporting/exporting.conf
@@ -17,6 +17,9 @@
 # [graphite:my_graphite_instance]
     # enabled = no
     # destination = localhost
+    # Credentials for basic HTTP authentication
+    # username = my_username
+    # password = my_password
     # data source = average
     # prefix = netdata
     # hostname = my_hostname
@@ -31,6 +34,8 @@
     # enabled = no
     # destination = localhost
     # remote write URL path = /receive
+    # username = my_username
+    # password = my_password
     # data source = average
     # prefix = netdata
     # hostname = my_hostname
diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h
index 1ad6e6856f..f08583fb54 100644
--- a/exporting/exporting_engine.h
+++ b/exporting/exporting_engine.h
@@ -66,6 +66,8 @@ struct instance_config {
 
     const char *name;
     const char *destination;
+    const char *username;
+    const char *password;
     const char *prefix;
     const char *hostname;
 
@@ -104,6 +106,8 @@ struct simple_connector_data {
     void *connector_specific_data;
 
     char connected_to[CONNECTED_TO_MAX];
+    
+    char *auth_string;
 
     size_t total_buffered_metrics;
 
diff --git a/exporting/graphite/README.md b/exporting/graphite/README.md
index a6a25ef7aa..d755e09345 100644
--- a/exporting/graphite/README.md
+++ b/exporting/graphite/README.md
@@ -22,7 +22,12 @@ directory and set the following options:
 ```
 
 Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `graphite:http:my_graphite_instance`,
-`graphite:https:my_graphite_instance`.
+`graphite:https:my_graphite_instance`. You can set basic HTTP authentication credentials using
+
+```conf
+    username = my_username
+    password = my_password
+```
 
 The Graphite connector is further configurable using additional settings. See the [exporting reference
 doc](/exporting/README.md#options) for details.
diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c
index 722db0fff2..84d4febf13 100644
--- a/exporting/graphite/graphite.c
+++ b/exporting/graphite/graphite.c
@@ -218,10 +218,12 @@ void graphite_http_prepare_header(struct instance *instance)
         simple_connector_data->last_buffer->header,
         "POST /api/put HTTP/1.1\r\n"
         "Host: %s\r\n"
+        "%s"
         "Content-Type: application/graphite\r\n"
         "Content-Length: %lu\r\n"
         "\r\n",
         instance->config.destination,
+        simple_connector_data->auth_string ? simple_connector_data->auth_string : "",
         buffer_strlen(simple_connector_data->last_buffer->buffer));
 
     return;
diff --git a/exporting/init_connectors.c b/exporting/init_connectors.c
index 6aff263549..6b22859bf7 100644
--- a/exporting/init_connectors.c
+++ b/exporting/init_connectors.c
@@ -105,8 +105,57 @@ int init_connectors(struct engine *engine)
     return 0;
 }
 
+// TODO: use a base64 encoder from a library
+static size_t base64_encode(unsigned char *input, size_t input_size, char *output, size_t output_size)
+{
+    uint32_t value;
+    static char lookup[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+                           "abcdefghijklmnopqrstuvwxyz"
+                           "0123456789+/";
+    if ((input_size / 3 + 1) * 4 >= output_size) {
+        error("Output buffer for encoding size=%zu is not large enough for %zu-bytes input", output_size, input_size);
+        return 0;
+    }
+    size_t count = 0;
+    while (input_size > 3) {
+        value = ((input[0] << 16) + (input[1] << 8) + input[2]) & 0xffffff;
+        output[0] = lookup[value >> 18];
+        output[1] = lookup[(value >> 12) & 0x3f];
+        output[2] = lookup[(value >> 6) & 0x3f];
+        output[3] = lookup[value & 0x3f];
+        //error("Base-64 encode (%04x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]);
+        output += 4;
+        input += 3;
+        input_size -= 3;
+        count += 4;
+    }
+    switch (input_size) {
+        case 2:
+            value = (input[0] << 10) + (input[1] << 2);
+            output[0] = lookup[(value >> 12) & 0x3f];
+            output[1] = lookup[(value >> 6) & 0x3f];
+            output[2] = lookup[value & 0x3f];
+            output[3] = '=';
+            //error("Base-64 encode (%06x) -> %c %c %c %c\n", (value>>2)&0xffff, output[0], output[1], output[2], output[3]);
+            count += 4;
+            break;
+        case 1:
+            value = input[0] << 4;
+            output[0] = lookup[(value >> 6) & 0x3f];
+            output[1] = lookup[value & 0x3f];
+            output[2] = '=';
+            output[3] = '=';
+            //error("Base-64 encode (%06x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]);
+            count += 4;
+            break;
+        case 0:
+            break;
+    }
+    return count;
+}
+
 /**
- * Initialize a ring buffer for a simple connector
+ * Initialize a ring buffer and credentials for a simple connector
  *
  * @param instance an instance data structure.
  */
@@ -141,5 +190,25 @@ void simple_connector_init(struct instance *instance)
     first_buffer->next = connector_specific_data->first_buffer;
     connector_specific_data->last_buffer = connector_specific_data->first_buffer;
 
+    if (*instance->config.username || *instance->config.password) {
+        BUFFER *auth_string = buffer_create(0);
+
+        buffer_sprintf(auth_string, "%s:%s", instance->config.username, instance->config.password);
+
+        size_t encoded_size = (buffer_strlen(auth_string) / 3 + 1) * 4 + 1;
+        char *encoded_credentials = callocz(1, encoded_size);
+
+        base64_encode((unsigned char*)buffer_tostring(auth_string), buffer_strlen(auth_string), encoded_credentials, encoded_size);
+       
+        buffer_flush(auth_string);
+        buffer_sprintf(auth_string, "Authorization: Basic %s\n", encoded_credentials);
+
+        freez(encoded_credentials);
+
+        connector_specific_data->auth_string = strdupz(buffer_tostring(auth_string));
+
+        buffer_free(auth_string);
+    }
+
     return;
 }
diff --git a/exporting/json/README.md b/exporting/json/README.md
index a0f8472a01..7cce463e20 100644
--- a/exporting/json/README.md
+++ b/exporting/json/README.md
@@ -22,7 +22,12 @@ directory and set the following options:
 ```
 
 Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `json:http:my_json_instance`,
-`json:https:my_json_instance`.
+`json:https:my_json_instance`. You can set basic HTTP authentication credentials using
+
+```conf
+    username = my_username
+    password = my_password
+```
 
 The JSON connector is further configurable using additional settings. See the [exporting reference
 doc](/exporting/README.md#options) for details.
diff --git a/exporting/json/json.c b/exporting/json/json.c
index f2396bafa1..50278c5b82 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -352,10 +352,12 @@ void json_http_prepare_header(struct instance *instance)
         simple_connector_data->last_buffer->header,
         "POST /api/put HTTP/1.1\r\n"
         "Host: %s\r\n"
+        "%s"
         "Content-Type: application/json\r\n"
         "Content-Length: %lu\r\n"
         "\r\n",
         instance->config.destination,
+        simple_connector_data->auth_string ? simple_connector_data->auth_string : "",
         buffer_strlen(simple_connector_data->last_buffer->buffer));
 
     return;
diff --git a/exporting/opentsdb/README.md b/exporting/opentsdb/README.md
index 3765ad2712..0ca6d2449c 100644
--- a/exporting/opentsdb/README.md
+++ b/exporting/opentsdb/README.md
@@ -22,7 +22,12 @@ directory and set the following options:
 ```
 
 Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `opentsdb:http:my_opentsdb_instance`,
-`opentsdb:https:my_opentsdb_instance`.
+`opentsdb:https:my_opentsdb_instance`. You can set basic HTTP authentication credentials using
+
+```conf
+    username = my_username
+    password = my_password
+```
 
 The OpenTSDB connector is further configurable using additional settings. See the [exporting reference
 doc](/exporting/README.md#options) for details.
diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c
index 1310c150e9..7ed88fd6d8 100644
--- a/exporting/opentsdb/opentsdb.c
+++ b/exporting/opentsdb/opentsdb.c
@@ -269,10 +269,12 @@ void opentsdb_http_prepare_header(struct instance *instance)
         simple_connector_data->last_buffer->header,
         "POST /api/put HTTP/1.1\r\n"
         "Host: %s\r\n"
+        "%s"
         "Content-Type: application/json\r\n"
         "Content-Length: %lu\r\n"
         "\r\n",
         instance->config.destination,
+        simple_connector_data->auth_string ? simple_connector_data->auth_string : "",
         buffer_strlen(simple_connector_data->last_buffer->buffer));
 
     return;
diff --git a/exporting/prometheus/remote_write/README.md b/exporting/prometheus/remote_write/README.md
index fe901024bd..ce379063e9 100644
--- a/exporting/prometheus/remote_write/README.md
+++ b/exporting/prometheus/remote_write/README.md
@@ -41,6 +41,13 @@ For example, if your endpoint is `http://example.domain:example_port/storage/rea
     remote write URL path = /storage/read
 ```
 
+You can set basic HTTP authentication credentials using
+
+```conf
+    username = my_username
+    password = my_password
+```
+
 `buffered` and `lost` dimensions in the Netdata Exporting Connector Data Size operation monitoring chart estimate uncompressed
 buffer size on failures.
 
diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c
index 986ad9f0ea..8339712eb6 100644
--- a/exporting/prometheus/remote_write/remote_write.c
+++ b/exporting/prometheus/remote_write/remote_write.c
@@ -25,6 +25,7 @@ void prometheus_remote_write_prepare_header(struct instance *instance)
         "POST %s HTTP/1.1\r\n"
         "Host: %s\r\n"
         "Accept: */*\r\n"
+        "%s"
         "Content-Encoding: snappy\r\n"
         "Content-Type: application/x-protobuf\r\n"
         "X-Prometheus-Remote-Write-Version: 0.1.0\r\n"
@@ -32,6 +33,7 @@ void prometheus_remote_write_prepare_header(struct instance *instance)
         "\r\n",
         connector_specific_config->remote_write_path,
         simple_connector_data->connected_to,
+        simple_connector_data->auth_string ? simple_connector_data->auth_string : "",
         buffer_strlen(simple_connector_data->last_buffer->buffer));
 }
 
diff --git a/exporting/read_config.c b/exporting/read_config.c
index ea50fa0f6f..77687d845f 100644
--- a/exporting/read_config.c
+++ b/exporting/read_config.c
@@ -456,6 +456,10 @@ struct engine *read_exporting_config()
 
         tmp_instance->config.destination = strdupz(exporter_get(instance_name, "destination", default_destination));
 
+        tmp_instance->config.username = strdupz(exporter_get(instance_name, "username", ""));
+
+        tmp_instance->config.password = strdupz(exporter_get(instance_name, "password", ""));
+
         tmp_instance->config.prefix = strdupz(exporter_get(instance_name, "prefix", "netdata"));
 
         tmp_instance->config.hostname = strdupz(exporter_get(instance_name, "hostname", engine->config.hostname));
diff --git a/exporting/tests/exporting_doubles.c b/exporting/tests/exporting_doubles.c
index 3c73e0327c..b8c9f37560 100644
--- a/exporting/tests/exporting_doubles.c
+++ b/exporting/tests/exporting_doubles.c
@@ -22,6 +22,8 @@ struct engine *__mock_read_exporting_config()
     instance->config.type = EXPORTING_CONNECTOR_TYPE_GRAPHITE;
     instance->config.name = strdupz("instance_name");
     instance->config.destination = strdupz("localhost");
+    instance->config.username = strdupz("");
+    instance->config.password = strdupz("");
     instance->config.prefix = strdupz("netdata");
     instance->config.hostname = strdupz("test-host");
     instance->config.update_every = 1;
diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c
index b5b0ce8165..b632761e7e 100644
--- a/exporting/tests/exporting_fixtures.c
+++ b/exporting/tests/exporting_fixtures.c
@@ -18,6 +18,8 @@ int teardown_configured_engine(void **state)
 
     struct instance *instance = engine->instance_root;
     free((void *)instance->config.destination);
+    free((void *)instance->config.username);
+    free((void *)instance->config.password);
     free((void *)instance->config.name);
     free((void *)instance->config.prefix);
     free((void *)instance->config.hostname);
diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c
index 73fd3ca668..845c96c3c9 100644
--- a/exporting/tests/test_exporting_engine.c
+++ b/exporting/tests/test_exporting_engine.c
@@ -1053,7 +1053,7 @@ static void test_format_host_labels_prometheus(void **state)
     instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
 
     format_host_labels_prometheus(instance, localhost);
-    assert_string_equal(buffer_tostring(instance->labels), "key1=\"netdata\",key2=\"value2\"");
+    assert_string_equal(buffer_tostring(instance->labels), "key1=\"value1\",key2=\"value2\"");
 }
 
 static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
@@ -1877,7 +1877,7 @@ int main(void)
         cmocka_unit_test_setup_teardown(test_prometheus_label_copy, setup_prometheus, teardown_prometheus),
         cmocka_unit_test_setup_teardown(test_prometheus_units_copy, setup_prometheus, teardown_prometheus),
         cmocka_unit_test_setup_teardown(
-            test_format_host_labels_prometheus, setup_configured_engine, teardown_configured_engine),
+            test_format_host_labels_prometheus, setup_initialized_engine, teardown_initialized_engine),
         cmocka_unit_test_setup_teardown(
             rrd_stats_api_v1_charts_allmetrics_prometheus, setup_prometheus, teardown_prometheus),
     };