Add HTTP and HTTPS support to the simple exporting connector (#9911)

3 files changed, 144 insertions, 4 deletions

diff --git a/exporting/json/README.md b/exporting/json/README.md
index 8ef084cf22..1dd41e738e 100644
--- a/exporting/json/README.md
+++ b/exporting/json/README.md
@@ -3,6 +3,7 @@ title: "Export metrics to JSON document databases"
 sidebar_label: JSON
 description: "Archive your Agent's metrics to a JSON document database for long-term storage, further analysis, or correlation with data from other sources."
 custom_edit_url: https://github.com/netdata/netdata/edit/master/exporting/json/README.md
+sidebar_label: JSON Document Databases
 -->
 
 # Export metrics to JSON document databases
@@ -21,6 +22,9 @@ directory and set the following options:
     destination = localhost:5448
 ```
 
+Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `json:http:my_json_instance`,
+`json:https:my_json_instance`.
+
 The JSON connector is further configurable using additional settings. See the [exporting reference
 doc](/exporting/README.md#options) for details.
 
diff --git a/exporting/json/json.c b/exporting/json/json.c
index cea7eff07c..458e7e8cce 100644
--- a/exporting/json/json.c
+++ b/exporting/json/json.c
@@ -16,6 +16,9 @@ int init_json_instance(struct instance *instance)
     instance->config.connector_specific_config = (void *)connector_specific_config;
     connector_specific_config->default_port = 5448;
 
+    struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
+    instance->connector_specific_data = connector_specific_data;
+
     instance->start_batch_formatting = NULL;
     instance->start_host_formatting = format_host_labels_json_plaintext;
     instance->start_chart_formatting = NULL;
@@ -27,9 +30,10 @@ int init_json_instance(struct instance *instance)
 
     instance->end_chart_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
-    instance->end_batch_formatting = simple_connector_update_buffered_bytes;
+    instance->end_batch_formatting = simple_connector_end_batch;
+
+    instance->prepare_header = NULL;
 
-    instance->send_header = NULL;
     instance->check_response = exporting_discard_response;
 
     instance->buffer = (void *)buffer_create(0);
@@ -37,6 +41,63 @@ int init_json_instance(struct instance *instance)
         error("EXPORTING: cannot create buffer for json exporting connector instance %s", instance->config.name);
         return 1;
     }
+
+    simple_connector_init(instance);
+
+    if (uv_mutex_init(&instance->mutex))
+        return 1;
+    if (uv_cond_init(&instance->cond_var))
+        return 1;
+
+    return 0;
+}
+
+/**
+ * Initialize JSON connector instance for HTTP protocol
+ *
+ * @param instance an instance data structure.
+ * @return Returns 0 on success, 1 on failure.
+ */
+int init_json_http_instance(struct instance *instance)
+{
+    instance->worker = simple_connector_worker;
+
+    struct simple_connector_config *connector_specific_config = callocz(1, sizeof(struct simple_connector_config));
+    instance->config.connector_specific_config = (void *)connector_specific_config;
+    connector_specific_config->default_port = 5448;
+
+    struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
+    instance->connector_specific_data = connector_specific_data;
+
+#ifdef ENABLE_HTTPS
+    connector_specific_data->flags = NETDATA_SSL_START;
+    connector_specific_data->conn = NULL;
+    if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
+        security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
+    }
+#endif
+
+    instance->start_batch_formatting = open_batch_json_http;
+    instance->start_host_formatting = format_host_labels_json_plaintext;
+    instance->start_chart_formatting = NULL;
+
+    if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
+        instance->metric_formatting = format_dimension_collected_json_plaintext;
+    else
+        instance->metric_formatting = format_dimension_stored_json_plaintext;
+
+    instance->end_chart_formatting = NULL;
+    instance->end_host_formatting = flush_host_labels;
+    instance->end_batch_formatting = close_batch_json_http;
+
+    instance->prepare_header = json_http_prepare_header;
+
+    instance->check_response = exporting_discard_response;
+
+    instance->buffer = (void *)buffer_create(0);
+
+    simple_connector_init(instance);
+
     if (uv_mutex_init(&instance->mutex))
         return 1;
     if (uv_cond_init(&instance->cond_var))
@@ -111,6 +172,11 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
         }
     }
 
+    if (instance->config.type == EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
+        if (buffer_strlen((BUFFER *)instance->buffer) > 2)
+        buffer_strcat(instance->buffer, ",\n");
+    }
+
     buffer_sprintf(
         instance->buffer,
 
@@ -131,7 +197,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
         "\"name\":\"%s\","
         "\"value\":" COLLECTED_NUMBER_FORMAT ","
 
-        "\"timestamp\":%llu}\n",
+        "\"timestamp\":%llu}",
 
         instance->config.prefix,
         (host == localhost) ? engine->config.hostname : host->hostname,
@@ -153,6 +219,10 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
 
         (unsigned long long)rd->last_collected_time.tv_sec);
 
+    if (instance->config.type != EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
+        buffer_strcat(instance->buffer, "\n");
+    }
+
     return 0;
 }
 
@@ -189,6 +259,11 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
         }
     }
 
+    if (instance->config.type == EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
+        if (buffer_strlen((BUFFER *)instance->buffer) > 2)
+            buffer_strcat(instance->buffer, ",\n");
+    }
+
     buffer_sprintf(
         instance->buffer,
         "{"
@@ -208,7 +283,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
         "\"name\":\"%s\","
         "\"value\":" CALCULATED_NUMBER_FORMAT ","
 
-        "\"timestamp\": %llu}\n",
+        "\"timestamp\": %llu}",
 
         instance->config.prefix,
         (host == localhost) ? engine->config.hostname : host->hostname,
@@ -230,5 +305,60 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
 
         (unsigned long long)last_t);
 
+    if (instance->config.type != EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
+        buffer_strcat(instance->buffer, "\n");
+    }
+
     return 0;
 }
+
+/**
+ * Open a JSON list for a bach
+ *
+ * @param instance an instance data structure.
+ * @return Always returns 0.
+ */
+int open_batch_json_http(struct instance *instance)
+{
+    buffer_strcat(instance->buffer, "[\n");
+
+    return 0;
+}
+
+/**
+ * Close a JSON list for a bach and update buffered bytes counter
+ *
+ * @param instance an instance data structure.
+ * @return Always returns 0.
+ */
+int close_batch_json_http(struct instance *instance)
+{
+    buffer_strcat(instance->buffer, "\n]\n");
+
+    simple_connector_end_batch(instance);
+
+    return 0;
+}
+
+/**
+ * Prepare HTTP header
+ *
+ * @param instance an instance data structure.
+ * @return Returns 0 on success, 1 on failure.
+ */
+void json_http_prepare_header(struct instance *instance)
+{
+    struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
+
+    buffer_sprintf(
+        simple_connector_data->last_buffer->header,
+        "POST /api/put HTTP/1.1\r\n"
+        "Host: %s\r\n"
+        "Content-Type: application/json\r\n"
+        "Content-Length: %lu\r\n"
+        "\r\n",
+        instance->config.destination,
+        buffer_strlen(simple_connector_data->last_buffer->buffer));
+
+    return;
+}
diff --git a/exporting/json/json.h b/exporting/json/json.h
index f4d3756631..d916263a9b 100644
--- a/exporting/json/json.h
+++ b/exporting/json/json.h
@@ -6,10 +6,16 @@
 #include "exporting/exporting_engine.h"
 
 int init_json_instance(struct instance *instance);
+int init_json_http_instance(struct instance *instance);
 
 int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host);
 
 int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM *rd);
 int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd);
 
+int open_batch_json_http(struct instance *instance);
+int close_batch_json_http(struct instance *instance);
+
+void json_http_prepare_header(struct instance *instance);
+
 #endif //NETDATA_EXPORTING_JSON_H