Prometheus web api connector (#8540)

* Fix the Prometheus web API code in the exporting engine

* Rename connector types

* Remove the conditional compilation of the exporting engine

* Use labels instead of tags

* Fix the exporter configuration

* Document functions

* Add unit tests

11 files changed, 533 insertions, 175 deletions

diff --git a/exporting/exporting_engine.h b/exporting/exporting_engine.h
index 1a3a3ecd09..8a46dd5d3d 100644
--- a/exporting/exporting_engine.h
+++ b/exporting/exporting_engine.h
@@ -14,10 +14,10 @@
 extern struct config exporting_config;
 
 #define EXPORTING_UPDATE_EVERY_OPTION_NAME "update every"
-#define EXPORTING_UPDATE_EVERY_DEFAULT     10
+#define EXPORTING_UPDATE_EVERY_DEFAULT 10
 
 typedef enum exporting_options {
-    EXPORTING_OPTION_NONE                   = 0,
+    EXPORTING_OPTION_NON                    = 0,
 
     EXPORTING_SOURCE_DATA_AS_COLLECTED      = (1 << 0),
     EXPORTING_SOURCE_DATA_AVERAGE           = (1 << 1),
@@ -42,10 +42,22 @@ typedef enum exporting_options {
      (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS &&                                             \
       label->label_source != LABEL_SOURCE_NETDATA_CONF))
 
+typedef enum exporting_connector_types {
+    EXPORTING_CONNECTOR_TYPE_UNKNOWN,                 // Invalid type
+    EXPORTING_CONNECTOR_TYPE_GRAPHITE,                // Send plain text to Graphite
+    EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET,   // Send data to OpenTSDB using telnet API
+    EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP,     // Send data to OpenTSDB using HTTP API
+    EXPORTING_CONNECTOR_TYPE_JSON,                    // Stores the data using JSON.
+    EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // The user selected to use Prometheus backend
+    EXPORTING_CONNECTOR_TYPE_KINESIS,                 // Send message to AWS Kinesis
+    EXPORTING_CONNECTOR_TYPE_MONGODB,                 // Send data to MongoDB collection
+    EXPORTING_CONNECTOR_TYPE_NUM                      // Number of backend types
+} EXPORTING_CONNECTOR_TYPE;
+
 struct engine;
 
 struct instance_config {
-    BACKEND_TYPE type;
+    EXPORTING_CONNECTOR_TYPE type;
 
     const char *name;
     const char *destination;
@@ -150,10 +162,12 @@ struct engine {
     struct instance *instance_root;
 };
 
+extern struct instance *prometheus_exporter_instance;
+
 void *exporting_main(void *ptr);
 
 struct engine *read_exporting_config();
-BACKEND_TYPE exporting_select_type(const char *type);
+EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type);
 
 int init_connectors(struct engine *engine);
 
@@ -187,4 +201,6 @@ void simple_connector_worker(void *instance_p);
 
 int send_internal_metrics(struct engine *engine);
 
+#include "exporting/prometheus/prometheus.h"
+
 #endif /* NETDATA_EXPORTING_ENGINE_H */
diff --git a/exporting/init_connectors.c b/exporting/init_connectors.c
index 798101fd9c..0db0ca1354 100644
--- a/exporting/init_connectors.c
+++ b/exporting/init_connectors.c
@@ -32,35 +32,35 @@ int init_connectors(struct engine *engine)
         instance->after = engine->now;
 
         switch (instance->config.type) {
-            case BACKEND_TYPE_GRAPHITE:
+            case EXPORTING_CONNECTOR_TYPE_GRAPHITE:
                 if (init_graphite_instance(instance) != 0)
                     return 1;
                 break;
-            case BACKEND_TYPE_JSON:
+            case EXPORTING_CONNECTOR_TYPE_JSON:
                 if (init_json_instance(instance) != 0)
                     return 1;
                 break;
-            case BACKEND_TYPE_OPENTSDB_USING_TELNET:
+            case EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET:
                 if (init_opentsdb_telnet_instance(instance) != 0)
                     return 1;
                 break;
-            case BACKEND_TYPE_OPENTSDB_USING_HTTP:
+            case EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP:
                 if (init_opentsdb_http_instance(instance) != 0)
                     return 1;
                 break;
-            case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE:
+            case EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE:
 #if ENABLE_PROMETHEUS_REMOTE_WRITE
                 if (init_prometheus_remote_write_instance(instance) != 0)
                     return 1;
 #endif
                 break;
-            case BACKEND_TYPE_KINESIS:
+            case EXPORTING_CONNECTOR_TYPE_KINESIS:
 #if HAVE_KINESIS
                 if (init_aws_kinesis_instance(instance) != 0)
                     return 1;
 #endif
                 break;
-            case BACKEND_TYPE_MONGODB:
+            case EXPORTING_CONNECTOR_TYPE_MONGODB:
 #if HAVE_MONGOC
                 if (init_mongodb_instance(instance) != 0)
                     return 1;
@@ -77,7 +77,7 @@ int init_connectors(struct engine *engine)
             error("EXPORTING: cannot create tread worker. uv_thread_create(): %s", uv_strerror(error));
             return 1;
         }
-        char threadname[NETDATA_THREAD_NAME_MAX+1];
+        char threadname[NETDATA_THREAD_NAME_MAX + 1];
         snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "EXPORTING-%zu", instance->index);
         uv_thread_set_name_np(instance->thread, threadname);
     }
diff --git a/exporting/mongodb/mongodb.c b/exporting/mongodb/mongodb.c
index b10a8fa664..bd2d338894 100644
--- a/exporting/mongodb/mongodb.c
+++ b/exporting/mongodb/mongodb.c
@@ -208,7 +208,7 @@ int format_batch_mongodb(struct instance *instance)
         insert[documents_inserted] = bson_new_from_json((const uint8_t *)start, -1, &bson_error);
 
         if (unlikely(!insert[documents_inserted])) {
-            error("EXPORTING: %s", bson_error.message);
+            error("EXPORTING: Failed creating a BSON document from a JSON string \"%s\" : %s", start, bson_error.message);
             free_bson(insert, documents_inserted);
             return 1;
         }
diff --git a/exporting/process_data.c b/exporting/process_data.c
index c902aabdda..0645982c8a 100644
--- a/exporting/process_data.c
+++ b/exporting/process_data.c
@@ -206,7 +206,7 @@ int start_host_formatting(struct engine *engine, RRDHOST *host)
  * Start chart formatting for every connector instance's buffer
  *
  * @param engine an engine data structure.
- * @param a chart.
+ * @param st a chart.
  * @return Returns 0 on success, 1 on failure.
  */
 int start_chart_formatting(struct engine *engine, RRDSET *st)
diff --git a/exporting/prometheus/prometheus.c b/exporting/prometheus/prometheus.c
index 7e91a0c962..f25ceed9d5 100644
--- a/exporting/prometheus/prometheus.c
+++ b/exporting/prometheus/prometheus.c
@@ -7,10 +7,16 @@
 // PROMETHEUS
 // /api/v1/allmetrics?format=prometheus and /api/v1/allmetrics?format=prometheus_all_hosts
 
+/**
+ * Check if a chart can be sent to an external databese
+ *
+ * @param instance an instance data structure.
+ * @param st a chart.
+ * @return Returns 1 if the chart can be sent, 0 otherwise.
+ */
 inline int can_send_rrdset(struct instance *instance, RRDSET *st)
 {
     RRDHOST *host = st->rrdhost;
-    (void)host;
 
     if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))
         return 0;
@@ -24,7 +30,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
             rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE);
             debug(
                 D_BACKEND,
-                "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.",
+                "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.",
                 st->id,
                 host->hostname);
             return 0;
@@ -34,7 +40,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
     if (unlikely(!rrdset_is_available_for_backends(st))) {
         debug(
             D_BACKEND,
-            "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.",
+            "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.",
             st->id,
             host->hostname);
         return 0;
@@ -42,10 +48,10 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)
 
     if (unlikely(
             st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&
-            !(BACKEND_OPTIONS_DATA_SOURCE(instance->config.options) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {
+            !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {
         debug(
             D_BACKEND,
-            "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.",
+            "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.",
             st->id,
             host->hostname,
             rrd_memory_mode_name(host->rrd_memory_mode));
@@ -63,8 +69,19 @@ static struct prometheus_server {
     struct prometheus_server *next;
 } *prometheus_server_root = NULL;
 
+/**
+ * Get the last time when a Prometheus server scraped the Netdata Prometheus exporter.
+ *
+ * @param server the name of the Prometheus server.
+ * @param host a data collecting host.
+ * @param now actual time.
+ * @return Returns the last time when the server accessed Netdata, or 0 if it is the first occurrence.
+ */
 static inline time_t prometheus_server_last_access(const char *server, RRDHOST *host, time_t now)
 {
+#ifdef UNIT_TESTING
+    return 0;
+#endif
     static netdata_mutex_t prometheus_server_root_mutex = NETDATA_MUTEX_INITIALIZER;
 
     uint32_t hash = simple_hash(server);
@@ -93,6 +110,14 @@ static inline time_t prometheus_server_last_access(const char *server, RRDHOST *
     return 0;
 }
 
+/**
+ * Copy and sanitize name.
+ *
+ * @param d a destination string.
+ * @param s a source sting.
+ * @param usable the number of characters to copy.
+ * @return Returns the length of the copied string.
+ */
 inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
 {
     size_t n;
@@ -110,6 +135,14 @@ inline size_t prometheus_name_copy(char *d, const char *s, size_t usable)
     return n;
 }
 
+/**
+ * Copy and sanitize label.
+ *
+ * @param d a destination string.
+ * @param s a source sting.
+ * @param usable the number of characters to copy.
+ * @return Returns the length of the copied string.
+ */
 inline size_t prometheus_label_copy(char *d, const char *s, size_t usable)
 {
     size_t n;
@@ -131,6 +164,15 @@ inline size_t prometheus_label_copy(char *d, const char *s, size_t usable)
     return n;
 }
 
+/**
+ * Copy and sanitize units.
+ *
+ * @param d a destination string.
+ * @param s a source sting.
+ * @param usable the number of characters to copy.
+ * @param showoldunits set this flag to 1 to show old (before v1.12) units.
+ * @return Returns the destination string.
+ */
 inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits)
 {
     const char *sorig = s;
@@ -203,6 +245,43 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh
     return ret;
 }
 
+/**
+ * Format host labels for the Prometheus exporter
+ *
+ * @param instance an instance data structure.
+ * @param host a data collecting host.
+ */
+void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
+{
+    if (unlikely(!sending_labels_configured(instance)))
+        return;
+
+    if (!instance->labels)
+        instance->labels = buffer_create(1024);
+
+    int count = 0;
+    rrdhost_check_rdlock(host);
+    netdata_rwlock_rdlock(&host->labels_rwlock);
+    for (struct label *label = host->labels; label; label = label->next) {
+        if (!should_send_label(instance, label))
+            continue;
+
+        char key[PROMETHEUS_ELEMENT_MAX + 1];
+        char value[PROMETHEUS_ELEMENT_MAX + 1];
+
+        prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
+        prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
+
+        if (*key && *value) {
+            if (count > 0)
+                buffer_strcat(instance->labels, ",");
+            buffer_sprintf(instance->labels, "%s=\"%s\"", key, value);
+            count++;
+        }
+    }
+    netdata_rwlock_unlock(&host->labels_rwlock);
+}
+
 struct host_variables_callback_options {
     RRDHOST *host;
     BUFFER *wb;
@@ -215,6 +294,13 @@ struct host_variables_callback_options {
     char name[PROMETHEUS_VARIABLE_MAX + 1];
 };
 
+/**
+ * Print host variables.
+ *
+ * @param rv a variable.
+ * @param data callback options.
+ * @return Returns 1 if the chart can be sent, 0 otherwise.
+ */
 static int print_host_variables(RRDVAR *rv, void *data)
 {
     struct host_variables_callback_options *opts = data;
@@ -274,14 +360,23 @@ static int print_host_variables(RRDVAR *rv, void *data)
     return 0;
 }
 
+/**
+ * Write metrics in Prometheus format to a buffer.
+ *
+ * @param instance an instance data structure.
+ * @param host a data collecting host.
+ * @param wb the buffer to fill with metrics.
+ * @param prefix a prefix for every metric.
+ * @param exporting_options options to configure what data is exported.
+ * @param allhosts set to 1 if host instance should be in the output for tags.
+ * @param output_options options to configure the format of the output.
+ */
 static void rrd_stats_api_v1_charts_allmetrics_prometheus(
     struct instance *instance,
     RRDHOST *host,
     BUFFER *wb,
     const char *prefix,
     EXPORTING_OPTIONS exporting_options,
-    time_t after,
-    time_t before,
     int allhosts,
     PROMETHEUS_OUTPUT_OPTIONS output_options)
 {
@@ -290,31 +385,33 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
     char hostname[PROMETHEUS_ELEMENT_MAX + 1];
     prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
 
+    format_host_labels_prometheus(instance, host);
+
+    if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
+        buffer_sprintf(
+            wb,
+            "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
+            hostname,
+            host->program_name,
+            host->program_version,
+            now_realtime_usec() / USEC_PER_MS);
+    else
+        buffer_sprintf(
+            wb,
+            "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
+            hostname,
+            host->program_name,
+            host->program_version);
+
     char labels[PROMETHEUS_LABELS_MAX + 1] = "";
     if (allhosts) {
-        if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
-            buffer_sprintf(
-                wb,
-                "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
-                hostname,
-                host->program_name,
-                host->program_version,
-                now_realtime_usec() / USEC_PER_MS);
-        else
-            buffer_sprintf(
-                wb,
-                "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
-                hostname,
-                host->program_name,
-                host->program_version);
-
-        if (host->tags && *(host->tags)) {
+        if (instance->labels && buffer_tostring(instance->labels)) {
             if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
                 buffer_sprintf(
                     wb,
                     "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
                     hostname,
-                    host->tags,
+                    buffer_tostring(instance->labels),
                     now_realtime_usec() / USEC_PER_MS);
 
                 // deprecated, exists only for compatibility with older queries
@@ -322,50 +419,46 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                     wb,
                     "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
                     hostname,
-                    host->tags,
+                    buffer_tostring(instance->labels),
                     now_realtime_usec() / USEC_PER_MS);
             } else {
-                buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, host->tags);
+                buffer_sprintf(
+                    wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
 
                 // deprecated, exists only for compatibility with older queries
-                buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, host->tags);
+                buffer_sprintf(
+                    wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
             }
         }
 
         snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
     } else {
-        if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)
-            buffer_sprintf(
-                wb,
-                "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n",
-                hostname,
-                host->program_name,
-                host->program_version,
-                now_realtime_usec() / USEC_PER_MS);
-        else
-            buffer_sprintf(
-                wb,
-                "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n",
-                hostname,
-                host->program_name,
-                host->program_version);
-
-        if (host->tags && *(host->tags)) {
+        if (instance->labels && buffer_tostring(instance->labels)) {
             if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
                 buffer_sprintf(
-                    wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
+                    wb,
+                    "netdata_host_tags_info{%s} 1 %llu\n",
+                    buffer_tostring(instance->labels),
+                    now_realtime_usec() / USEC_PER_MS);
 
                 // deprecated, exists only for compatibility with older queries
-                buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);
+                buffer_sprintf(
+                    wb,
+                    "netdata_host_tags{%s} 1 %llu\n",
+                    buffer_tostring(instance->labels),
+                    now_realtime_usec() / USEC_PER_MS);
             } else {
-                buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", host->tags);
+                buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels));
 
                 // deprecated, exists only for compatibility with older queries
-                buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", host->tags);
+                buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels));
             }
         }
     }
 
+    if (instance->labels)
+        buffer_flush(instance->labels);
+
     // send custom variables set for the host
     if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
         struct host_variables_callback_options opts = { .host = host,
@@ -383,20 +476,20 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
     RRDSET *st;
     rrdset_foreach_read(st, host)
     {
-        char chart[PROMETHEUS_ELEMENT_MAX + 1];
-        char context[PROMETHEUS_ELEMENT_MAX + 1];
-        char family[PROMETHEUS_ELEMENT_MAX + 1];
-
-        prometheus_label_copy(
-            chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX);
-        prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
-        prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
 
         if (likely(can_send_rrdset(instance, st))) {
             rrdset_rdlock(st);
 
+            char chart[PROMETHEUS_ELEMENT_MAX + 1];
+            char context[PROMETHEUS_ELEMENT_MAX + 1];
+            char family[PROMETHEUS_ELEMENT_MAX + 1];
             char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
 
+            prometheus_label_copy(
+                chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id, PROMETHEUS_ELEMENT_MAX);
+            prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);
+            prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);
+
             int as_collected = (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AS_COLLECTED);
             int homogeneous = 1;
             if (as_collected) {
@@ -433,7 +526,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                     if (as_collected) {
                         // we need as-collected / raw data
 
-                        if (unlikely(rd->last_collected_time.tv_sec < after))
+                        if (unlikely(rd->last_collected_time.tv_sec < instance->after))
                             continue;
 
                         const char *t = "gauge", *h = "gives";
@@ -562,8 +655,9 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                     } else {
                         // we need average or sum of the data
 
-                        time_t first_t = after, last_t = before;
-                        calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_t);
+                        time_t first_time = instance->after;
+                        time_t last_time = instance->before;
+                        calculated_number value = exporting_calculate_value_from_stored_data(instance, rd, &last_time);
 
                         if (!isnan(value) && !isinf(value)) {
                             if (EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) == EXPORTING_SOURCE_DATA_AVERAGE)
@@ -586,8 +680,8 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                                     suffix,
                                     (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id,
                                     st->units,
-                                    (unsigned long long)first_t,
-                                    (unsigned long long)last_t);
+                                    (unsigned long long)first_time,
+                                    (unsigned long long)last_time);
 
                             if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))
                                 buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n", prefix, context, units, suffix);
@@ -606,7 +700,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
                                     dimension,
                                     labels,
                                     value,
-                                    last_t * MSEC_PER_SEC);
+                                    last_time * MSEC_PER_SEC);
                             else
                                 buffer_sprintf(
                                     wb,
@@ -633,6 +727,18 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
     rrdhost_unlock(host);
 }
 
+/**
+ * Get the last time time when a server accessed Netdata. Write information about an API request to a buffer.
+ *
+ * @param instance an instance data structure.
+ * @param host a data collecting host.
+ * @param wb the buffer to write to.
+ * @param exporting_options options to configure what data is exported.
+ * @param server the name of a Prometheus server..
+ * @param now actual time.
+ * @param output_options options to configure the format of the output.
+ * @return Returns the last time when the server accessed Netdata.
+ */
 static inline time_t prometheus_preparation(
     struct instance *instance,
     RRDHOST *host,
@@ -649,13 +755,13 @@ static inline time_t prometheus_preparation(
 
     int first_seen = 0;
     if (!after) {
-        after = now - instance->engine->config.update_every;
+        after = now - instance->config.update_every;
         first_seen = 1;
     }
 
     if (after > now) {
         // oops! this should never happen
-        after = now - instance->engine->config.update_every;
+        after = now - instance->config.update_every;
     }
 
     if (output_options & PROMETHEUS_OUTPUT_HELP) {
@@ -685,8 +791,17 @@ static inline time_t prometheus_preparation(
     return after;
 }
 
+/**
+ * Write metrics and auxiliary information for one host to a buffer.
+ *
+ * @param host a data collecting host.
+ * @param wb the buffer to write to.
+ * @param server the name of a Prometheus server.
+ * @param prefix a prefix for every metric.
+ * @param exporting_options options to configure what data is exported.
+ * @param output_options options to configure the format of the output.
+ */
 void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
-    struct instance *instance,
     RRDHOST *host,
     BUFFER *wb,
     const char *server,
@@ -694,17 +809,36 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
     EXPORTING_OPTIONS exporting_options,
     PROMETHEUS_OUTPUT_OPTIONS output_options)
 {
-    time_t before = now_realtime_sec();
+    if (unlikely(!prometheus_exporter_instance))
+        return;
+
+    prometheus_exporter_instance->before = now_realtime_sec();
 
     // we start at the point we had stopped before
-    time_t after = prometheus_preparation(instance, host, wb, exporting_options, server, before, output_options);
+    prometheus_exporter_instance->after = prometheus_preparation(
+        prometheus_exporter_instance,
+        host,
+        wb,
+        exporting_options,
+        server,
+        prometheus_exporter_instance->before,
+        output_options);
 
     rrd_stats_api_v1_charts_allmetrics_prometheus(
-        instance, host, wb, prefix, exporting_options, after, before, 0, output_options);
+        prometheus_exporter_instance, host, wb, prefix, exporting_options, 0, output_options);
 }
 
+/**
+ * Write metrics and auxiliary information for all hosts to a buffer.
+ *
+ * @param host a data collecting host.
+ * @param wb the buffer to write to.
+ * @param server the name of a Prometheus server.
+ * @param prefix a prefix for every metric.
+ * @param exporting_options options to configure what data is exported.
+ * @param output_options options to configure the format of the output.
+ */
 void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
-    struct instance *instance,
     RRDHOST *host,
     BUFFER *wb,
     const char *server,
@@ -712,16 +846,26 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
     EXPORTING_OPTIONS exporting_options,
     PROMETHEUS_OUTPUT_OPTIONS output_options)
 {
-    time_t before = now_realtime_sec();
+    if (unlikely(!prometheus_exporter_instance))
+        return;
+
+    prometheus_exporter_instance->before = now_realtime_sec();
 
     // we start at the point we had stopped before
-    time_t after = prometheus_preparation(instance, host, wb, exporting_options, server, before, output_options);
+    prometheus_exporter_instance->after = prometheus_preparation(
+        prometheus_exporter_instance,
+        host,
+        wb,
+        exporting_options,
+        server,
+        prometheus_exporter_instance->before,
+        output_options);
 
     rrd_rdlock();
     rrdhost_foreach_read(host)
     {
         rrd_stats_api_v1_charts_allmetrics_prometheus(
-            instance, host, wb, prefix, exporting_options, after, before, 1, output_options);
+            prometheus_exporter_instance, host, wb, prefix, exporting_options, 1, output_options);
     }
     rrd_unlock();
 }
diff --git a/exporting/prometheus/prometheus.h b/exporting/prometheus/prometheus.h
index b947633deb..85bcc7a7f8 100644
--- a/exporting/prometheus/prometheus.h
+++ b/exporting/prometheus/prometheus.h
@@ -23,10 +23,10 @@ typedef enum prometheus_output_flags {
 } PROMETHEUS_OUTPUT_OPTIONS;
 
 extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
-    struct instance *instance, RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
+    RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
     EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
 extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
-    struct instance *instance, RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
+    RRDHOST *host, BUFFER *wb, const char *server, const char *prefix,
     EXPORTING_OPTIONS exporting_options, PROMETHEUS_OUTPUT_OPTIONS output_options);
 
 int can_send_rrdset(struct instance *instance, RRDSET *st);
@@ -34,4 +34,6 @@ size_t prometheus_name_copy(char *d, const char *s, size_t usable);
 size_t prometheus_label_copy(char *d, const char *s, size_t usable);
 char *prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits);
 
+void format_host_labels_prometheus(struct instance *instance, RRDHOST *host);
+
 #endif //NETDATA_EXPORTING_PROMETHEUS_H
diff --git a/exporting/read_config.c b/exporting/read_config.c
index a9590659e0..122ceef79d 100644
--- a/exporting/read_config.c
+++ b/exporting/read_config.c
@@ -2,18 +2,19 @@
 
 #include "exporting_engine.h"
 
-struct config exporting_config = {.first_section = NULL,
-                                  .last_section = NULL,
-                                  .mutex = NETDATA_MUTEX_INITIALIZER,
-                                  .index = {.avl_tree = {.root = NULL, .compar = appconfig_section_compare},
-                                            .rwlock = AVL_LOCK_INITIALIZER}};
+struct config exporting_config = { .first_section = NULL,
+                                   .last_section = NULL,
+                                   .mutex = NETDATA_MUTEX_INITIALIZER,
+                                   .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
+                                              .rwlock = AVL_LOCK_INITIALIZER } };
 
+struct instance *prometheus_exporter_instance = NULL;
 
 static _CONNECTOR_INSTANCE *find_instance(const char *section)
 {
     _CONNECTOR_INSTANCE *local_ci;
 
-    local_ci = add_connector_instance(NULL, NULL);  // Get root section
+    local_ci = add_connector_instance(NULL, NULL); // Get root section
     if (unlikely(!local_ci))
         return local_ci;
 
@@ -38,12 +39,10 @@ char *expconfig_get(struct config *root, const char *section, const char *name,
     local_ci = find_instance(section);
 
     if (!local_ci)
-        return NULL;    // TODO: Check if it is meaningful to return default_value
+        return NULL; // TODO: Check if it is meaningful to return default_value
 
     return appconfig_get(
-        root,
-        local_ci->instance_name,
-        name,
+        root, local_ci->instance_name, name,
         appconfig_get(
             root, local_ci->connector_name, name, appconfig_get(root, CONFIG_SECTION_EXPORTING, name, default_value)));
 }
@@ -58,16 +57,12 @@ int expconfig_get_boolean(struct config *root, const char *section, const char *
     local_ci = find_instance(section);
 
     if (!local_ci)
-        return 0;       // TODO: Check if it is meaningful to return default_value
+        return 0; // TODO: Check if it is meaningful to return default_value
 
     return appconfig_get_boolean(
-        root,
-        local_ci->instance_name,
-        name,
+        root, local_ci->instance_name, name,
         appconfig_get_boolean(
-            root,
-            local_ci->connector_name,
-            name,
+            root, local_ci->connector_name, name,
             appconfig_get_boolean(root, CONFIG_SECTION_EXPORTING, name, default_value)));
 }
 
@@ -81,16 +76,12 @@ long long expconfig_get_number(struct config *root, const char *section, const c
     local_ci = find_instance(section);
 
     if (!local_ci)
-        return 0;   // TODO: Check if it is meaningful to return default_value
+        return 0; // TODO: Check if it is meaningful to return default_value
 
     return appconfig_get_number(
-        root,
-        local_ci->instance_name,
-        name,
+        root, local_ci->instance_name, name,
         appconfig_get_number(
-            root,
-            local_ci->connector_name,
-            name,
+            root, local_ci->connector_name, name,
             appconfig_get_number(root, CONFIG_SECTION_EXPORTING, name, default_value)));
 }
 
@@ -108,7 +99,7 @@ int get_connector_instance(struct connector_instance *target_ci)
     static _CONNECTOR_INSTANCE *local_ci = NULL;
     _CONNECTOR_INSTANCE *global_connector_instance;
 
-    global_connector_instance = find_instance(NULL);       // Fetch head of instances
+    global_connector_instance = find_instance(NULL); // Fetch head of instances
 
     if (unlikely(!global_connector_instance))
         return 0;
@@ -140,40 +131,39 @@ int get_connector_instance(struct connector_instance *target_ci)
  *
  * @return It returns the connector id.
  */
-BACKEND_TYPE exporting_select_type(const char *type)
+EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type)
 {