From 416e9f4a4a4d5f395592f3aaa8212755bbb5851e Mon Sep 17 00:00:00 2001 From: Vladimir Kobal Date: Wed, 4 Aug 2021 16:26:27 +0300 Subject: Add HTTP basic authentication to some exporting connectors (#11394) --- exporting/README.md | 2 + exporting/clean_connectors.c | 4 ++ exporting/exporting.conf | 5 ++ exporting/exporting_engine.h | 4 ++ exporting/graphite/README.md | 7 ++- exporting/graphite/graphite.c | 2 + exporting/init_connectors.c | 71 +++++++++++++++++++++++- exporting/json/README.md | 7 ++- exporting/json/json.c | 2 + exporting/opentsdb/README.md | 7 ++- exporting/opentsdb/opentsdb.c | 2 + exporting/prometheus/remote_write/README.md | 7 +++ exporting/prometheus/remote_write/remote_write.c | 2 + exporting/read_config.c | 4 ++ exporting/tests/exporting_doubles.c | 2 + exporting/tests/exporting_fixtures.c | 2 + exporting/tests/test_exporting_engine.c | 4 +- 17 files changed, 128 insertions(+), 6 deletions(-) (limited to 'exporting') diff --git a/exporting/README.md b/exporting/README.md index 933de0e075..ef485bb186 100644 --- a/exporting/README.md +++ b/exporting/README.md @@ -164,6 +164,8 @@ You can configure each connector individually using the available [options](#opt [opentsdb:http:my_opentsdb_http_instance] enabled = yes destination = localhost:4242 + username = my_username + password = my_password [opentsdb:https:my_opentsdb_https_instance] enabled = yes diff --git a/exporting/clean_connectors.c b/exporting/clean_connectors.c index 890e8daac7..4af1219a63 100644 --- a/exporting/clean_connectors.c +++ b/exporting/clean_connectors.c @@ -15,6 +15,8 @@ static void clean_instance_config(struct instance_config *config) freez((void *)config->type_name); freez((void *)config->name); freez((void *)config->destination); + freez((void *)config->username); + freez((void *)config->password); freez((void *)config->prefix); freez((void *)config->hostname); @@ -49,6 +51,8 @@ void simple_connector_cleanup(struct instance *instance) struct simple_connector_data *simple_connector_data = (struct simple_connector_data *)instance->connector_specific_data; + freez(simple_connector_data->auth_string); + buffer_free(instance->buffer); buffer_free(simple_connector_data->buffer); buffer_free(simple_connector_data->header); diff --git a/exporting/exporting.conf b/exporting/exporting.conf index c2e902c051..314e1541ef 100644 --- a/exporting/exporting.conf +++ b/exporting/exporting.conf @@ -17,6 +17,9 @@ # [graphite:my_graphite_instance] # enabled = no # destination = localhost + # Credentials for basic HTTP authentication + # username = my_username + # password = my_password # data source = average # prefix = netdata # hostname = my_hostname @@ -31,6 +34,8 @@ # enabled = no # destination = localhost # remote write URL path = /receive + # username = my_username + # password = my_password # data source = average # prefix = netdata # hostname = my_hostname diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h index 1ad6e6856f..f08583fb54 100644 --- a/exporting/exporting_engine.h +++ b/exporting/exporting_engine.h @@ -66,6 +66,8 @@ struct instance_config { const char *name; const char *destination; + const char *username; + const char *password; const char *prefix; const char *hostname; @@ -104,6 +106,8 @@ struct simple_connector_data { void *connector_specific_data; char connected_to[CONNECTED_TO_MAX]; + + char *auth_string; size_t total_buffered_metrics; diff --git a/exporting/graphite/README.md b/exporting/graphite/README.md index a6a25ef7aa..d755e09345 100644 --- a/exporting/graphite/README.md +++ b/exporting/graphite/README.md @@ -22,7 +22,12 @@ directory and set the following options: ``` Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `graphite:http:my_graphite_instance`, -`graphite:https:my_graphite_instance`. +`graphite:https:my_graphite_instance`. You can set basic HTTP authentication credentials using + +```conf + username = my_username + password = my_password +``` The Graphite connector is further configurable using additional settings. See the [exporting reference doc](/exporting/README.md#options) for details. diff --git a/exporting/graphite/graphite.c b/exporting/graphite/graphite.c index 722db0fff2..84d4febf13 100644 --- a/exporting/graphite/graphite.c +++ b/exporting/graphite/graphite.c @@ -218,10 +218,12 @@ void graphite_http_prepare_header(struct instance *instance) simple_connector_data->last_buffer->header, "POST /api/put HTTP/1.1\r\n" "Host: %s\r\n" + "%s" "Content-Type: application/graphite\r\n" "Content-Length: %lu\r\n" "\r\n", instance->config.destination, + simple_connector_data->auth_string ? simple_connector_data->auth_string : "", buffer_strlen(simple_connector_data->last_buffer->buffer)); return; diff --git a/exporting/init_connectors.c b/exporting/init_connectors.c index 6aff263549..6b22859bf7 100644 --- a/exporting/init_connectors.c +++ b/exporting/init_connectors.c @@ -105,8 +105,57 @@ int init_connectors(struct engine *engine) return 0; } +// TODO: use a base64 encoder from a library +static size_t base64_encode(unsigned char *input, size_t input_size, char *output, size_t output_size) +{ + uint32_t value; + static char lookup[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + if ((input_size / 3 + 1) * 4 >= output_size) { + error("Output buffer for encoding size=%zu is not large enough for %zu-bytes input", output_size, input_size); + return 0; + } + size_t count = 0; + while (input_size > 3) { + value = ((input[0] << 16) + (input[1] << 8) + input[2]) & 0xffffff; + output[0] = lookup[value >> 18]; + output[1] = lookup[(value >> 12) & 0x3f]; + output[2] = lookup[(value >> 6) & 0x3f]; + output[3] = lookup[value & 0x3f]; + //error("Base-64 encode (%04x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]); + output += 4; + input += 3; + input_size -= 3; + count += 4; + } + switch (input_size) { + case 2: + value = (input[0] << 10) + (input[1] << 2); + output[0] = lookup[(value >> 12) & 0x3f]; + output[1] = lookup[(value >> 6) & 0x3f]; + output[2] = lookup[value & 0x3f]; + output[3] = '='; + //error("Base-64 encode (%06x) -> %c %c %c %c\n", (value>>2)&0xffff, output[0], output[1], output[2], output[3]); + count += 4; + break; + case 1: + value = input[0] << 4; + output[0] = lookup[(value >> 6) & 0x3f]; + output[1] = lookup[value & 0x3f]; + output[2] = '='; + output[3] = '='; + //error("Base-64 encode (%06x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]); + count += 4; + break; + case 0: + break; + } + return count; +} + /** - * Initialize a ring buffer for a simple connector + * Initialize a ring buffer and credentials for a simple connector * * @param instance an instance data structure. */ @@ -141,5 +190,25 @@ void simple_connector_init(struct instance *instance) first_buffer->next = connector_specific_data->first_buffer; connector_specific_data->last_buffer = connector_specific_data->first_buffer; + if (*instance->config.username || *instance->config.password) { + BUFFER *auth_string = buffer_create(0); + + buffer_sprintf(auth_string, "%s:%s", instance->config.username, instance->config.password); + + size_t encoded_size = (buffer_strlen(auth_string) / 3 + 1) * 4 + 1; + char *encoded_credentials = callocz(1, encoded_size); + + base64_encode((unsigned char*)buffer_tostring(auth_string), buffer_strlen(auth_string), encoded_credentials, encoded_size); + + buffer_flush(auth_string); + buffer_sprintf(auth_string, "Authorization: Basic %s\n", encoded_credentials); + + freez(encoded_credentials); + + connector_specific_data->auth_string = strdupz(buffer_tostring(auth_string)); + + buffer_free(auth_string); + } + return; } diff --git a/exporting/json/README.md b/exporting/json/README.md index a0f8472a01..7cce463e20 100644 --- a/exporting/json/README.md +++ b/exporting/json/README.md @@ -22,7 +22,12 @@ directory and set the following options: ``` Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `json:http:my_json_instance`, -`json:https:my_json_instance`. +`json:https:my_json_instance`. You can set basic HTTP authentication credentials using + +```conf + username = my_username + password = my_password +``` The JSON connector is further configurable using additional settings. See the [exporting reference doc](/exporting/README.md#options) for details. diff --git a/exporting/json/json.c b/exporting/json/json.c index f2396bafa1..50278c5b82 100644 --- a/exporting/json/json.c +++ b/exporting/json/json.c @@ -352,10 +352,12 @@ void json_http_prepare_header(struct instance *instance) simple_connector_data->last_buffer->header, "POST /api/put HTTP/1.1\r\n" "Host: %s\r\n" + "%s" "Content-Type: application/json\r\n" "Content-Length: %lu\r\n" "\r\n", instance->config.destination, + simple_connector_data->auth_string ? simple_connector_data->auth_string : "", buffer_strlen(simple_connector_data->last_buffer->buffer)); return; diff --git a/exporting/opentsdb/README.md b/exporting/opentsdb/README.md index 3765ad2712..0ca6d2449c 100644 --- a/exporting/opentsdb/README.md +++ b/exporting/opentsdb/README.md @@ -22,7 +22,12 @@ directory and set the following options: ``` Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `opentsdb:http:my_opentsdb_instance`, -`opentsdb:https:my_opentsdb_instance`. +`opentsdb:https:my_opentsdb_instance`. You can set basic HTTP authentication credentials using + +```conf + username = my_username + password = my_password +``` The OpenTSDB connector is further configurable using additional settings. See the [exporting reference doc](/exporting/README.md#options) for details. diff --git a/exporting/opentsdb/opentsdb.c b/exporting/opentsdb/opentsdb.c index 1310c150e9..7ed88fd6d8 100644 --- a/exporting/opentsdb/opentsdb.c +++ b/exporting/opentsdb/opentsdb.c @@ -269,10 +269,12 @@ void opentsdb_http_prepare_header(struct instance *instance) simple_connector_data->last_buffer->header, "POST /api/put HTTP/1.1\r\n" "Host: %s\r\n" + "%s" "Content-Type: application/json\r\n" "Content-Length: %lu\r\n" "\r\n", instance->config.destination, + simple_connector_data->auth_string ? simple_connector_data->auth_string : "", buffer_strlen(simple_connector_data->last_buffer->buffer)); return; diff --git a/exporting/prometheus/remote_write/README.md b/exporting/prometheus/remote_write/README.md index fe901024bd..ce379063e9 100644 --- a/exporting/prometheus/remote_write/README.md +++ b/exporting/prometheus/remote_write/README.md @@ -41,6 +41,13 @@ For example, if your endpoint is `http://example.domain:example_port/storage/rea remote write URL path = /storage/read ``` +You can set basic HTTP authentication credentials using + +```conf + username = my_username + password = my_password +``` + `buffered` and `lost` dimensions in the Netdata Exporting Connector Data Size operation monitoring chart estimate uncompressed buffer size on failures. diff --git a/exporting/prometheus/remote_write/remote_write.c b/exporting/prometheus/remote_write/remote_write.c index 986ad9f0ea..8339712eb6 100644 --- a/exporting/prometheus/remote_write/remote_write.c +++ b/exporting/prometheus/remote_write/remote_write.c @@ -25,6 +25,7 @@ void prometheus_remote_write_prepare_header(struct instance *instance) "POST %s HTTP/1.1\r\n" "Host: %s\r\n" "Accept: */*\r\n" + "%s" "Content-Encoding: snappy\r\n" "Content-Type: application/x-protobuf\r\n" "X-Prometheus-Remote-Write-Version: 0.1.0\r\n" @@ -32,6 +33,7 @@ void prometheus_remote_write_prepare_header(struct instance *instance) "\r\n", connector_specific_config->remote_write_path, simple_connector_data->connected_to, + simple_connector_data->auth_string ? simple_connector_data->auth_string : "", buffer_strlen(simple_connector_data->last_buffer->buffer)); } diff --git a/exporting/read_config.c b/exporting/read_config.c index ea50fa0f6f..77687d845f 100644 --- a/exporting/read_config.c +++ b/exporting/read_config.c @@ -456,6 +456,10 @@ struct engine *read_exporting_config() tmp_instance->config.destination = strdupz(exporter_get(instance_name, "destination", default_destination)); + tmp_instance->config.username = strdupz(exporter_get(instance_name, "username", "")); + + tmp_instance->config.password = strdupz(exporter_get(instance_name, "password", "")); + tmp_instance->config.prefix = strdupz(exporter_get(instance_name, "prefix", "netdata")); tmp_instance->config.hostname = strdupz(exporter_get(instance_name, "hostname", engine->config.hostname)); diff --git a/exporting/tests/exporting_doubles.c b/exporting/tests/exporting_doubles.c index 3c73e0327c..b8c9f37560 100644 --- a/exporting/tests/exporting_doubles.c +++ b/exporting/tests/exporting_doubles.c @@ -22,6 +22,8 @@ struct engine *__mock_read_exporting_config() instance->config.type = EXPORTING_CONNECTOR_TYPE_GRAPHITE; instance->config.name = strdupz("instance_name"); instance->config.destination = strdupz("localhost"); + instance->config.username = strdupz(""); + instance->config.password = strdupz(""); instance->config.prefix = strdupz("netdata"); instance->config.hostname = strdupz("test-host"); instance->config.update_every = 1; diff --git a/exporting/tests/exporting_fixtures.c b/exporting/tests/exporting_fixtures.c index b5b0ce8165..b632761e7e 100644 --- a/exporting/tests/exporting_fixtures.c +++ b/exporting/tests/exporting_fixtures.c @@ -18,6 +18,8 @@ int teardown_configured_engine(void **state) struct instance *instance = engine->instance_root; free((void *)instance->config.destination); + free((void *)instance->config.username); + free((void *)instance->config.password); free((void *)instance->config.name); free((void *)instance->config.prefix); free((void *)instance->config.hostname); diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 73fd3ca668..845c96c3c9 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -1053,7 +1053,7 @@ static void test_format_host_labels_prometheus(void **state) instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS; format_host_labels_prometheus(instance, localhost); - assert_string_equal(buffer_tostring(instance->labels), "key1=\"netdata\",key2=\"value2\""); + assert_string_equal(buffer_tostring(instance->labels), "key1=\"value1\",key2=\"value2\""); } static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) @@ -1877,7 +1877,7 @@ int main(void) cmocka_unit_test_setup_teardown(test_prometheus_label_copy, setup_prometheus, teardown_prometheus), cmocka_unit_test_setup_teardown(test_prometheus_units_copy, setup_prometheus, teardown_prometheus), cmocka_unit_test_setup_teardown( - test_format_host_labels_prometheus, setup_configured_engine, teardown_configured_engine), + test_format_host_labels_prometheus, setup_initialized_engine, teardown_initialized_engine), cmocka_unit_test_setup_teardown( rrd_stats_api_v1_charts_allmetrics_prometheus, setup_prometheus, teardown_prometheus), }; -- cgit v1.2.3