Show internal stats for the exporting engine (#8635)

* Add a print function for internal exporting statistics

* Send statistics for simple connectors

* Flush sending buffers on failures

* Send statistics for the Kinesis connector

* Send statistics for the MongoDB connector

* Add unit tests

1 files changed, 27 insertions, 9 deletions

diff --git a/exporting/aws_kinesis/aws_kinesis.c b/exporting/aws_kinesis/aws_kinesis.c
index 4b0d5f74a4..938569a9eb 100644
--- a/exporting/aws_kinesis/aws_kinesis.c
+++ b/exporting/aws_kinesis/aws_kinesis.c
@@ -75,9 +75,23 @@ void aws_kinesis_connector_worker(void *instance_p)
         uv_mutex_lock(&instance->mutex);
         uv_cond_wait(&instance->cond_var, &instance->mutex);
 
+        // reset the monitoring chart counters
+        stats->received_bytes =
+        stats->sent_bytes =
+        stats->sent_metrics =
+        stats->lost_metrics =
+        stats->receptions =
+        stats->transmission_successes =
+        stats->transmission_failures =
+        stats->data_lost_events =
+        stats->lost_bytes =
+        stats->reconnects = 0;
+
         BUFFER *buffer = (BUFFER *)instance->buffer;
         size_t buffer_len = buffer_strlen(buffer);
 
+        stats->buffered_bytes = buffer_len;
+
         size_t sent = 0;
 
         while (sent < buffer_len) {
@@ -115,7 +129,7 @@ void aws_kinesis_connector_worker(void *instance_p)
                 connector_specific_data, connector_specific_config->stream_name, partition_key, first_char, record_len);
 
             sent += record_len;
-            stats->chart_transmission_successes++;
+            stats->transmission_successes++;
 
             size_t sent_bytes = 0, lost_bytes = 0;
 
@@ -127,30 +141,34 @@ void aws_kinesis_connector_worker(void *instance_p)
                     "EXPORTING: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",
                     instance->config.destination, sent_bytes, sent_bytes - lost_bytes);
 
-                stats->chart_transmission_failures++;
-                stats->chart_data_lost_events++;
-                stats->chart_lost_bytes += lost_bytes;
+                stats->transmission_failures++;
+                stats->data_lost_events++;
+                stats->lost_bytes += lost_bytes;
 
                 // estimate the number of lost metrics
-                stats->chart_lost_metrics += (collected_number)(
-                    stats->chart_buffered_metrics *
+                stats->lost_metrics += (collected_number)(
+                    stats->buffered_metrics *
                     (buffer_len && (lost_bytes > buffer_len) ? (double)lost_bytes / buffer_len : 1));
 
                 break;
             } else {
-                stats->chart_receptions++;
+                stats->receptions++;
             }
 
             if (unlikely(netdata_exit))
                 break;
         }
 
-        stats->chart_sent_bytes += sent;
+        stats->sent_bytes += sent;
         if (likely(sent == buffer_len))
-            stats->chart_sent_metrics = stats->chart_buffered_metrics;
+            stats->sent_metrics = stats->buffered_metrics;
 
         buffer_flush(buffer);
 
+        send_internal_metrics(instance);
+
+        stats->buffered_metrics = 0;
+
         uv_mutex_unlock(&instance->mutex);
 
 #ifdef UNIT_TESTING