From 8121b18d1e59c1a28fb54844a01677fe9f5d29e2 Mon Sep 17 00:00:00 2001 From: vkalintiris Date: Mon, 5 Feb 2024 11:16:59 +0200 Subject: Move exporting/ under src/ (#16913) --- .github/labeler.yml | 2 +- CMakeLists.txt | 54 +- .../integrations-overview.md | 2 +- docs/export/enable-connector.md | 16 +- docs/export/external-databases.md | 80 +- docs/guides/using-host-labels.md | 4 +- exporting/README.md | 327 ------- exporting/TIMESCALE.md | 71 -- exporting/WALKTHROUGH.md | 260 ----- exporting/aws_kinesis/README.md | 1 - exporting/aws_kinesis/aws_kinesis.c | 219 ----- exporting/aws_kinesis/aws_kinesis.h | 16 - exporting/aws_kinesis/aws_kinesis_put_record.cc | 151 --- exporting/aws_kinesis/aws_kinesis_put_record.h | 35 - exporting/aws_kinesis/integrations/aws_kinesis.md | 168 ---- exporting/aws_kinesis/metadata.yaml | 173 ---- exporting/check_filters.c | 88 -- exporting/clean_connectors.c | 82 -- exporting/exporting.conf | 96 -- exporting/exporting_engine.c | 219 ----- exporting/exporting_engine.h | 322 ------ exporting/graphite/README.md | 1 - exporting/graphite/graphite.c | 219 ----- exporting/graphite/graphite.h | 18 - exporting/graphite/integrations/blueflood.md | 172 ---- exporting/graphite/integrations/graphite.md | 172 ---- exporting/graphite/integrations/influxdb.md | 172 ---- exporting/graphite/integrations/kairosdb.md | 172 ---- exporting/graphite/metadata.yaml | 212 ---- exporting/init_connectors.c | 219 ----- exporting/json/README.md | 1 - exporting/json/integrations/json.md | 147 --- exporting/json/json.c | 349 ------- exporting/json/json.h | 21 - exporting/json/metadata.yaml | 151 --- exporting/mongodb/README.md | 1 - exporting/mongodb/integrations/mongodb.md | 145 --- exporting/mongodb/metadata.yaml | 151 --- exporting/mongodb/mongodb.c | 392 -------- exporting/mongodb/mongodb.h | 35 - exporting/nc-exporting.sh | 158 --- exporting/opentsdb/README.md | 1 - exporting/opentsdb/integrations/opentsdb.md | 175 ---- exporting/opentsdb/metadata.yaml | 176 ---- exporting/opentsdb/opentsdb.c | 399 -------- exporting/opentsdb/opentsdb.h | 26 - exporting/process_data.c | 445 --------- exporting/prometheus/README.md | 361 ------- exporting/prometheus/integrations/appoptics.md | 158 --- .../prometheus/integrations/azure_data_explorer.md | 158 --- .../prometheus/integrations/azure_event_hub.md | 158 --- exporting/prometheus/integrations/chronix.md | 158 --- exporting/prometheus/integrations/cortex.md | 158 --- exporting/prometheus/integrations/cratedb.md | 158 --- exporting/prometheus/integrations/elasticsearch.md | 158 --- exporting/prometheus/integrations/gnocchi.md | 158 --- .../prometheus/integrations/google_bigquery.md | 158 --- exporting/prometheus/integrations/irondb.md | 158 --- exporting/prometheus/integrations/kafka.md | 158 --- exporting/prometheus/integrations/m3db.md | 158 --- exporting/prometheus/integrations/metricfire.md | 158 --- exporting/prometheus/integrations/new_relic.md | 158 --- exporting/prometheus/integrations/postgresql.md | 158 --- .../integrations/prometheus_remote_write.md | 158 --- exporting/prometheus/integrations/quasardb.md | 158 --- .../prometheus/integrations/splunk_signalfx.md | 158 --- exporting/prometheus/integrations/thanos.md | 158 --- exporting/prometheus/integrations/tikv.md | 158 --- exporting/prometheus/integrations/timescaledb.md | 158 --- .../prometheus/integrations/victoriametrics.md | 158 --- exporting/prometheus/integrations/vmware_aria.md | 158 --- exporting/prometheus/integrations/wavefront.md | 158 --- exporting/prometheus/metadata.yaml | 436 --------- exporting/prometheus/prometheus.c | 1023 -------------------- exporting/prometheus/prometheus.h | 39 - exporting/prometheus/remote_write/README.md | 1 - exporting/prometheus/remote_write/remote_write.c | 403 -------- exporting/prometheus/remote_write/remote_write.h | 32 - .../prometheus/remote_write/remote_write.proto | 29 - .../remote_write/remote_write_request.cc | 258 ----- .../prometheus/remote_write/remote_write_request.h | 42 - exporting/pubsub/README.md | 1 - .../pubsub/integrations/google_cloud_pub_sub.md | 145 --- exporting/pubsub/metadata.yaml | 152 --- exporting/pubsub/pubsub.c | 195 ---- exporting/pubsub/pubsub.h | 14 - exporting/pubsub/pubsub_publish.cc | 258 ----- exporting/pubsub/pubsub_publish.h | 37 - exporting/read_config.c | 515 ---------- exporting/sample-metadata.yaml | 39 - exporting/send_data.c | 400 -------- exporting/send_internal_metrics.c | 200 ---- integrations/integrations.js | 2 +- integrations/integrations.json | 2 +- src/exporting/README.md | 327 +++++++ src/exporting/TIMESCALE.md | 71 ++ src/exporting/WALKTHROUGH.md | 260 +++++ src/exporting/aws_kinesis/README.md | 1 + src/exporting/aws_kinesis/aws_kinesis.c | 219 +++++ src/exporting/aws_kinesis/aws_kinesis.h | 16 + .../aws_kinesis/aws_kinesis_put_record.cc | 151 +++ src/exporting/aws_kinesis/aws_kinesis_put_record.h | 35 + .../aws_kinesis/integrations/aws_kinesis.md | 168 ++++ src/exporting/aws_kinesis/metadata.yaml | 173 ++++ src/exporting/check_filters.c | 88 ++ src/exporting/clean_connectors.c | 82 ++ src/exporting/exporting.conf | 96 ++ src/exporting/exporting_engine.c | 219 +++++ src/exporting/exporting_engine.h | 322 ++++++ src/exporting/graphite/README.md | 1 + src/exporting/graphite/graphite.c | 219 +++++ src/exporting/graphite/graphite.h | 18 + src/exporting/graphite/integrations/blueflood.md | 172 ++++ src/exporting/graphite/integrations/graphite.md | 172 ++++ src/exporting/graphite/integrations/influxdb.md | 172 ++++ src/exporting/graphite/integrations/kairosdb.md | 172 ++++ src/exporting/graphite/metadata.yaml | 212 ++++ src/exporting/init_connectors.c | 219 +++++ src/exporting/json/README.md | 1 + src/exporting/json/integrations/json.md | 147 +++ src/exporting/json/json.c | 349 +++++++ src/exporting/json/json.h | 21 + src/exporting/json/metadata.yaml | 151 +++ src/exporting/mongodb/README.md | 1 + src/exporting/mongodb/integrations/mongodb.md | 145 +++ src/exporting/mongodb/metadata.yaml | 151 +++ src/exporting/mongodb/mongodb.c | 392 ++++++++ src/exporting/mongodb/mongodb.h | 35 + src/exporting/nc-exporting.sh | 158 +++ src/exporting/opentsdb/README.md | 1 + src/exporting/opentsdb/integrations/opentsdb.md | 175 ++++ src/exporting/opentsdb/metadata.yaml | 176 ++++ src/exporting/opentsdb/opentsdb.c | 399 ++++++++ src/exporting/opentsdb/opentsdb.h | 26 + src/exporting/process_data.c | 445 +++++++++ src/exporting/prometheus/README.md | 361 +++++++ src/exporting/prometheus/integrations/appoptics.md | 158 +++ .../prometheus/integrations/azure_data_explorer.md | 158 +++ .../prometheus/integrations/azure_event_hub.md | 158 +++ src/exporting/prometheus/integrations/chronix.md | 158 +++ src/exporting/prometheus/integrations/cortex.md | 158 +++ src/exporting/prometheus/integrations/cratedb.md | 158 +++ .../prometheus/integrations/elasticsearch.md | 158 +++ src/exporting/prometheus/integrations/gnocchi.md | 158 +++ .../prometheus/integrations/google_bigquery.md | 158 +++ src/exporting/prometheus/integrations/irondb.md | 158 +++ src/exporting/prometheus/integrations/kafka.md | 158 +++ src/exporting/prometheus/integrations/m3db.md | 158 +++ .../prometheus/integrations/metricfire.md | 158 +++ src/exporting/prometheus/integrations/new_relic.md | 158 +++ .../prometheus/integrations/postgresql.md | 158 +++ .../integrations/prometheus_remote_write.md | 158 +++ src/exporting/prometheus/integrations/quasardb.md | 158 +++ .../prometheus/integrations/splunk_signalfx.md | 158 +++ src/exporting/prometheus/integrations/thanos.md | 158 +++ src/exporting/prometheus/integrations/tikv.md | 158 +++ .../prometheus/integrations/timescaledb.md | 158 +++ .../prometheus/integrations/victoriametrics.md | 158 +++ .../prometheus/integrations/vmware_aria.md | 158 +++ src/exporting/prometheus/integrations/wavefront.md | 158 +++ src/exporting/prometheus/metadata.yaml | 436 +++++++++ src/exporting/prometheus/prometheus.c | 1023 ++++++++++++++++++++ src/exporting/prometheus/prometheus.h | 39 + src/exporting/prometheus/remote_write/README.md | 1 + .../prometheus/remote_write/remote_write.c | 403 ++++++++ .../prometheus/remote_write/remote_write.h | 32 + .../prometheus/remote_write/remote_write.proto | 29 + .../remote_write/remote_write_request.cc | 258 +++++ .../prometheus/remote_write/remote_write_request.h | 42 + src/exporting/pubsub/README.md | 1 + .../pubsub/integrations/google_cloud_pub_sub.md | 145 +++ src/exporting/pubsub/metadata.yaml | 152 +++ src/exporting/pubsub/pubsub.c | 195 ++++ src/exporting/pubsub/pubsub.h | 14 + src/exporting/pubsub/pubsub_publish.cc | 258 +++++ src/exporting/pubsub/pubsub_publish.h | 37 + src/exporting/read_config.c | 515 ++++++++++ src/exporting/sample-metadata.yaml | 39 + src/exporting/send_data.c | 400 ++++++++ src/exporting/send_internal_metrics.c | 200 ++++ web/api/exporters/prometheus/README.md | 2 +- web/gui/v2/2934.810bd676d65e8a58defb.chunk.js | 2 +- 182 files changed, 14813 insertions(+), 14813 deletions(-) delete mode 100644 exporting/README.md delete mode 100644 exporting/TIMESCALE.md delete mode 100644 exporting/WALKTHROUGH.md delete mode 120000 exporting/aws_kinesis/README.md delete mode 100644 exporting/aws_kinesis/aws_kinesis.c delete mode 100644 exporting/aws_kinesis/aws_kinesis.h delete mode 100644 exporting/aws_kinesis/aws_kinesis_put_record.cc delete mode 100644 exporting/aws_kinesis/aws_kinesis_put_record.h delete mode 100644 exporting/aws_kinesis/integrations/aws_kinesis.md delete mode 100644 exporting/aws_kinesis/metadata.yaml delete mode 100644 exporting/check_filters.c delete mode 100644 exporting/clean_connectors.c delete mode 100644 exporting/exporting.conf delete mode 100644 exporting/exporting_engine.c delete mode 100644 exporting/exporting_engine.h delete mode 120000 exporting/graphite/README.md delete mode 100644 exporting/graphite/graphite.c delete mode 100644 exporting/graphite/graphite.h delete mode 100644 exporting/graphite/integrations/blueflood.md delete mode 100644 exporting/graphite/integrations/graphite.md delete mode 100644 exporting/graphite/integrations/influxdb.md delete mode 100644 exporting/graphite/integrations/kairosdb.md delete mode 100644 exporting/graphite/metadata.yaml delete mode 100644 exporting/init_connectors.c delete mode 120000 exporting/json/README.md delete mode 100644 exporting/json/integrations/json.md delete mode 100644 exporting/json/json.c delete mode 100644 exporting/json/json.h delete mode 100644 exporting/json/metadata.yaml delete mode 120000 exporting/mongodb/README.md delete mode 100644 exporting/mongodb/integrations/mongodb.md delete mode 100644 exporting/mongodb/metadata.yaml delete mode 100644 exporting/mongodb/mongodb.c delete mode 100644 exporting/mongodb/mongodb.h delete mode 100755 exporting/nc-exporting.sh delete mode 120000 exporting/opentsdb/README.md delete mode 100644 exporting/opentsdb/integrations/opentsdb.md delete mode 100644 exporting/opentsdb/metadata.yaml delete mode 100644 exporting/opentsdb/opentsdb.c delete mode 100644 exporting/opentsdb/opentsdb.h delete mode 100644 exporting/process_data.c delete mode 100644 exporting/prometheus/README.md delete mode 100644 exporting/prometheus/integrations/appoptics.md delete mode 100644 exporting/prometheus/integrations/azure_data_explorer.md delete mode 100644 exporting/prometheus/integrations/azure_event_hub.md delete mode 100644 exporting/prometheus/integrations/chronix.md delete mode 100644 exporting/prometheus/integrations/cortex.md delete mode 100644 exporting/prometheus/integrations/cratedb.md delete mode 100644 exporting/prometheus/integrations/elasticsearch.md delete mode 100644 exporting/prometheus/integrations/gnocchi.md delete mode 100644 exporting/prometheus/integrations/google_bigquery.md delete mode 100644 exporting/prometheus/integrations/irondb.md delete mode 100644 exporting/prometheus/integrations/kafka.md delete mode 100644 exporting/prometheus/integrations/m3db.md delete mode 100644 exporting/prometheus/integrations/metricfire.md delete mode 100644 exporting/prometheus/integrations/new_relic.md delete mode 100644 exporting/prometheus/integrations/postgresql.md delete mode 100644 exporting/prometheus/integrations/prometheus_remote_write.md delete mode 100644 exporting/prometheus/integrations/quasardb.md delete mode 100644 exporting/prometheus/integrations/splunk_signalfx.md delete mode 100644 exporting/prometheus/integrations/thanos.md delete mode 100644 exporting/prometheus/integrations/tikv.md delete mode 100644 exporting/prometheus/integrations/timescaledb.md delete mode 100644 exporting/prometheus/integrations/victoriametrics.md delete mode 100644 exporting/prometheus/integrations/vmware_aria.md delete mode 100644 exporting/prometheus/integrations/wavefront.md delete mode 100644 exporting/prometheus/metadata.yaml delete mode 100644 exporting/prometheus/prometheus.c delete mode 100644 exporting/prometheus/prometheus.h delete mode 120000 exporting/prometheus/remote_write/README.md delete mode 100644 exporting/prometheus/remote_write/remote_write.c delete mode 100644 exporting/prometheus/remote_write/remote_write.h delete mode 100644 exporting/prometheus/remote_write/remote_write.proto delete mode 100644 exporting/prometheus/remote_write/remote_write_request.cc delete mode 100644 exporting/prometheus/remote_write/remote_write_request.h delete mode 120000 exporting/pubsub/README.md delete mode 100644 exporting/pubsub/integrations/google_cloud_pub_sub.md delete mode 100644 exporting/pubsub/metadata.yaml delete mode 100644 exporting/pubsub/pubsub.c delete mode 100644 exporting/pubsub/pubsub.h delete mode 100644 exporting/pubsub/pubsub_publish.cc delete mode 100644 exporting/pubsub/pubsub_publish.h delete mode 100644 exporting/read_config.c delete mode 100644 exporting/sample-metadata.yaml delete mode 100644 exporting/send_data.c delete mode 100644 exporting/send_internal_metrics.c create mode 100644 src/exporting/README.md create mode 100644 src/exporting/TIMESCALE.md create mode 100644 src/exporting/WALKTHROUGH.md create mode 120000 src/exporting/aws_kinesis/README.md create mode 100644 src/exporting/aws_kinesis/aws_kinesis.c create mode 100644 src/exporting/aws_kinesis/aws_kinesis.h create mode 100644 src/exporting/aws_kinesis/aws_kinesis_put_record.cc create mode 100644 src/exporting/aws_kinesis/aws_kinesis_put_record.h create mode 100644 src/exporting/aws_kinesis/integrations/aws_kinesis.md create mode 100644 src/exporting/aws_kinesis/metadata.yaml create mode 100644 src/exporting/check_filters.c create mode 100644 src/exporting/clean_connectors.c create mode 100644 src/exporting/exporting.conf create mode 100644 src/exporting/exporting_engine.c create mode 100644 src/exporting/exporting_engine.h create mode 120000 src/exporting/graphite/README.md create mode 100644 src/exporting/graphite/graphite.c create mode 100644 src/exporting/graphite/graphite.h create mode 100644 src/exporting/graphite/integrations/blueflood.md create mode 100644 src/exporting/graphite/integrations/graphite.md create mode 100644 src/exporting/graphite/integrations/influxdb.md create mode 100644 src/exporting/graphite/integrations/kairosdb.md create mode 100644 src/exporting/graphite/metadata.yaml create mode 100644 src/exporting/init_connectors.c create mode 120000 src/exporting/json/README.md create mode 100644 src/exporting/json/integrations/json.md create mode 100644 src/exporting/json/json.c create mode 100644 src/exporting/json/json.h create mode 100644 src/exporting/json/metadata.yaml create mode 120000 src/exporting/mongodb/README.md create mode 100644 src/exporting/mongodb/integrations/mongodb.md create mode 100644 src/exporting/mongodb/metadata.yaml create mode 100644 src/exporting/mongodb/mongodb.c create mode 100644 src/exporting/mongodb/mongodb.h create mode 100755 src/exporting/nc-exporting.sh create mode 120000 src/exporting/opentsdb/README.md create mode 100644 src/exporting/opentsdb/integrations/opentsdb.md create mode 100644 src/exporting/opentsdb/metadata.yaml create mode 100644 src/exporting/opentsdb/opentsdb.c create mode 100644 src/exporting/opentsdb/opentsdb.h create mode 100644 src/exporting/process_data.c create mode 100644 src/exporting/prometheus/README.md create mode 100644 src/exporting/prometheus/integrations/appoptics.md create mode 100644 src/exporting/prometheus/integrations/azure_data_explorer.md create mode 100644 src/exporting/prometheus/integrations/azure_event_hub.md create mode 100644 src/exporting/prometheus/integrations/chronix.md create mode 100644 src/exporting/prometheus/integrations/cortex.md create mode 100644 src/exporting/prometheus/integrations/cratedb.md create mode 100644 src/exporting/prometheus/integrations/elasticsearch.md create mode 100644 src/exporting/prometheus/integrations/gnocchi.md create mode 100644 src/exporting/prometheus/integrations/google_bigquery.md create mode 100644 src/exporting/prometheus/integrations/irondb.md create mode 100644 src/exporting/prometheus/integrations/kafka.md create mode 100644 src/exporting/prometheus/integrations/m3db.md create mode 100644 src/exporting/prometheus/integrations/metricfire.md create mode 100644 src/exporting/prometheus/integrations/new_relic.md create mode 100644 src/exporting/prometheus/integrations/postgresql.md create mode 100644 src/exporting/prometheus/integrations/prometheus_remote_write.md create mode 100644 src/exporting/prometheus/integrations/quasardb.md create mode 100644 src/exporting/prometheus/integrations/splunk_signalfx.md create mode 100644 src/exporting/prometheus/integrations/thanos.md create mode 100644 src/exporting/prometheus/integrations/tikv.md create mode 100644 src/exporting/prometheus/integrations/timescaledb.md create mode 100644 src/exporting/prometheus/integrations/victoriametrics.md create mode 100644 src/exporting/prometheus/integrations/vmware_aria.md create mode 100644 src/exporting/prometheus/integrations/wavefront.md create mode 100644 src/exporting/prometheus/metadata.yaml create mode 100644 src/exporting/prometheus/prometheus.c create mode 100644 src/exporting/prometheus/prometheus.h create mode 120000 src/exporting/prometheus/remote_write/README.md create mode 100644 src/exporting/prometheus/remote_write/remote_write.c create mode 100644 src/exporting/prometheus/remote_write/remote_write.h create mode 100644 src/exporting/prometheus/remote_write/remote_write.proto create mode 100644 src/exporting/prometheus/remote_write/remote_write_request.cc create mode 100644 src/exporting/prometheus/remote_write/remote_write_request.h create mode 120000 src/exporting/pubsub/README.md create mode 100644 src/exporting/pubsub/integrations/google_cloud_pub_sub.md create mode 100644 src/exporting/pubsub/metadata.yaml create mode 100644 src/exporting/pubsub/pubsub.c create mode 100644 src/exporting/pubsub/pubsub.h create mode 100644 src/exporting/pubsub/pubsub_publish.cc create mode 100644 src/exporting/pubsub/pubsub_publish.h create mode 100644 src/exporting/read_config.c create mode 100644 src/exporting/sample-metadata.yaml create mode 100644 src/exporting/send_data.c create mode 100644 src/exporting/send_internal_metrics.c diff --git a/.github/labeler.yml b/.github/labeler.yml index 6d3f603f2c..7ee7612bbd 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -28,7 +28,7 @@ area/exporting: - any: - changed-files: - any-glob-to-any-file: - - exporting/** + - src/exporting/** area/build: - any: diff --git a/CMakeLists.txt b/CMakeLists.txt index b51ffe29db..a40b0f141a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -922,23 +922,23 @@ set(API_PLUGIN_FILES ) set(EXPORTING_ENGINE_FILES - exporting/exporting_engine.c - exporting/exporting_engine.h - exporting/graphite/graphite.c - exporting/graphite/graphite.h - exporting/json/json.c - exporting/json/json.h - exporting/opentsdb/opentsdb.c - exporting/opentsdb/opentsdb.h - exporting/prometheus/prometheus.c - exporting/prometheus/prometheus.h - exporting/read_config.c - exporting/clean_connectors.c - exporting/init_connectors.c - exporting/process_data.c - exporting/check_filters.c - exporting/send_data.c - exporting/send_internal_metrics.c + src/exporting/exporting_engine.c + src/exporting/exporting_engine.h + src/exporting/graphite/graphite.c + src/exporting/graphite/graphite.h + src/exporting/json/json.c + src/exporting/json/json.h + src/exporting/opentsdb/opentsdb.c + src/exporting/opentsdb/opentsdb.h + src/exporting/prometheus/prometheus.c + src/exporting/prometheus/prometheus.h + src/exporting/read_config.c + src/exporting/clean_connectors.c + src/exporting/init_connectors.c + src/exporting/process_data.c + src/exporting/check_filters.c + src/exporting/send_data.c + src/exporting/send_internal_metrics.c ) set(HEALTH_PLUGIN_FILES @@ -1437,15 +1437,15 @@ set(ACLK_FILES set(MONGODB_EXPORTING_FILES - exporting/mongodb/mongodb.c - exporting/mongodb/mongodb.h + src/exporting/mongodb/mongodb.c + src/exporting/mongodb/mongodb.h ) set(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES - exporting/prometheus/remote_write/remote_write.c - exporting/prometheus/remote_write/remote_write.h - exporting/prometheus/remote_write/remote_write_request.cc - exporting/prometheus/remote_write/remote_write_request.h + src/exporting/prometheus/remote_write/remote_write.c + src/exporting/prometheus/remote_write/remote_write.h + src/exporting/prometheus/remote_write/remote_write_request.cc + src/exporting/prometheus/remote_write/remote_write_request.h ) # @@ -2062,11 +2062,11 @@ if(ENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE) endif() endif() - protoc_generate_cpp("${CMAKE_SOURCE_DIR}/exporting/prometheus/remote_write" - "${CMAKE_SOURCE_DIR}/exporting/prometheus/remote_write" + protoc_generate_cpp("${CMAKE_SOURCE_DIR}/src/exporting/prometheus/remote_write" + "${CMAKE_SOURCE_DIR}/src/exporting/prometheus/remote_write" PROMETHEUS_REMOTE_WRITE_BUILT_SRCS PROMETHEUS_REMOTE_WRITE_BUILT_HDRS - "exporting/prometheus/remote_write/remote_write.proto") + "src/exporting/prometheus/remote_write/remote_write.proto") list(APPEND PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES ${PROMETHEUS_REMOTE_WRITE_BUILT_SRCS} @@ -2313,7 +2313,7 @@ install(FILES # exporting # install(FILES - exporting/exporting.conf + src/exporting/exporting.conf DESTINATION usr/lib/netdata/conf.d) # diff --git a/docs/category-overview-pages/integrations-overview.md b/docs/category-overview-pages/integrations-overview.md index 9d9276a99b..322d00cca7 100644 --- a/docs/category-overview-pages/integrations-overview.md +++ b/docs/category-overview-pages/integrations-overview.md @@ -26,6 +26,6 @@ We designed Netdata with interoperability in mind. The Agent collects thousands you do with them is up to you. You can [store metrics in the database engine](https://github.com/netdata/netdata/blob/master/src/database/README.md), or send them to another time series database for long-term storage or further analysis using -Netdata's [exporting engine](https://github.com/netdata/netdata/edit/master/exporting/README.md). +Netdata's [exporting engine](https://github.com/netdata/netdata/edit/master/src/exporting/README.md). diff --git a/docs/export/enable-connector.md b/docs/export/enable-connector.md index 02e380e15d..dd3a55a4bf 100644 --- a/docs/export/enable-connector.md +++ b/docs/export/enable-connector.md @@ -60,8 +60,8 @@ Netdata Agent exports metrics _beginning from the time the process starts_, and collected, you should start seeing data in your external database after only a few seconds. Any further configuration is optional, based on your needs and the configuration of your OpenTSDB database. See the -[OpenTSDB connector doc](https://github.com/netdata/netdata/blob/master/exporting/opentsdb/README.md) -and [exporting engine reference](https://github.com/netdata/netdata/blob/master/exporting/README.md#configuration) for +[OpenTSDB connector doc](https://github.com/netdata/netdata/blob/master/src/exporting/opentsdb/README.md) +and [exporting engine reference](https://github.com/netdata/netdata/blob/master/src/exporting/README.md#configuration) for details. ## Example: Enable the Graphite connector @@ -84,22 +84,22 @@ Because the Agent exports metrics as they're collected, you should start seeing only a few seconds. Any further configuration is optional, based on your needs and the configuration of your Graphite-supported database. -See [exporting engine reference](https://github.com/netdata/netdata/blob/master/exporting/README.md#configuration) for +See [exporting engine reference](https://github.com/netdata/netdata/blob/master/src/exporting/README.md#configuration) for details. ## What's next? If you want to further configure your exporting connectors, see -the [exporting engine reference](https://github.com/netdata/netdata/blob/master/exporting/README.md#configuration). +the [exporting engine reference](https://github.com/netdata/netdata/blob/master/src/exporting/README.md#configuration). For a comprehensive example of using the Graphite connector, read our documentation on -[exporting metrics to Graphite providers](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md). Or, start +[exporting metrics to Graphite providers](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md). Or, start [using host labels](https://github.com/netdata/netdata/blob/master/docs/guides/using-host-labels.md) on exported metrics. ### Related reference documentation -- [Exporting engine reference](https://github.com/netdata/netdata/blob/master/exporting/README.md) -- [OpenTSDB connector](https://github.com/netdata/netdata/blob/master/exporting/opentsdb/README.md) -- [Graphite connector](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md) +- [Exporting engine reference](https://github.com/netdata/netdata/blob/master/src/exporting/README.md) +- [OpenTSDB connector](https://github.com/netdata/netdata/blob/master/src/exporting/opentsdb/README.md) +- [Graphite connector](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md) diff --git a/docs/export/external-databases.md b/docs/export/external-databases.md index 715e8660dc..1679947ce0 100644 --- a/docs/export/external-databases.md +++ b/docs/export/external-databases.md @@ -11,7 +11,7 @@ learn_rel_path: "Concepts" # Export metrics to external time-series databases Netdata allows you to export metrics to external time-series databases with the [exporting -engine](https://github.com/netdata/netdata/blob/master/exporting/README.md). This system uses a number of **connectors** to initiate connections to [more than +engine](https://github.com/netdata/netdata/blob/master/src/exporting/README.md). This system uses a number of **connectors** to initiate connections to [more than thirty](#supported-databases) supported databases, including InfluxDB, Prometheus, Graphite, ElasticSearch, and much more. @@ -29,48 +29,48 @@ analysis, or correlation with other tools, such as application tracing. ## Supported databases Netdata supports exporting metrics to the following databases through several -[connectors](https://github.com/netdata/netdata/blob/master/exporting/README.md#features). Once you find the connector that works for your database, open its +[connectors](https://github.com/netdata/netdata/blob/master/src/exporting/README.md#features). Once you find the connector that works for your database, open its documentation and the [enabling a connector](https://github.com/netdata/netdata/blob/master/docs/export/enable-connector.md) doc for details on enabling it. -- **AppOptics**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **AWS Kinesis**: [AWS Kinesis Data Streams](https://github.com/netdata/netdata/blob/master/exporting/aws_kinesis/README.md) -- **Azure Data Explorer**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Azure Event Hubs**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Blueflood**: [Graphite](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md) -- **Chronix**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Cortex**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **CrateDB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **ElasticSearch**: [Graphite](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md), [Prometheus remote - write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Gnocchi**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Google BigQuery**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Google Cloud Pub/Sub**: [Google Cloud Pub/Sub Service](https://github.com/netdata/netdata/blob/master/exporting/pubsub/README.md) -- **Graphite**: [Graphite](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md), [Prometheus remote - write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **InfluxDB**: [Graphite](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md), [Prometheus remote - write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **IRONdb**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **JSON**: [JSON document databases](https://github.com/netdata/netdata/blob/master/exporting/json/README.md) -- **Kafka**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **KairosDB**: [Graphite](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md), [OpenTSDB](https://github.com/netdata/netdata/blob/master/exporting/opentsdb/README.md) -- **M3DB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **MetricFire**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **MongoDB**: [MongoDB](https://github.com/netdata/netdata/blob/master/exporting/mongodb/README.md) -- **New Relic**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **OpenTSDB**: [OpenTSDB](https://github.com/netdata/netdata/blob/master/exporting/opentsdb/README.md), [Prometheus remote - write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **PostgreSQL**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) +- **AppOptics**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **AWS Kinesis**: [AWS Kinesis Data Streams](https://github.com/netdata/netdata/blob/master/src/exporting/aws_kinesis/README.md) +- **Azure Data Explorer**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Azure Event Hubs**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Blueflood**: [Graphite](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md) +- **Chronix**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Cortex**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **CrateDB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **ElasticSearch**: [Graphite](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md), [Prometheus remote + write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Gnocchi**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Google BigQuery**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Google Cloud Pub/Sub**: [Google Cloud Pub/Sub Service](https://github.com/netdata/netdata/blob/master/src/exporting/pubsub/README.md) +- **Graphite**: [Graphite](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md), [Prometheus remote + write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **InfluxDB**: [Graphite](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md), [Prometheus remote + write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **IRONdb**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **JSON**: [JSON document databases](https://github.com/netdata/netdata/blob/master/src/exporting/json/README.md) +- **Kafka**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **KairosDB**: [Graphite](https://github.com/netdata/netdata/blob/master/src/exporting/graphite/README.md), [OpenTSDB](https://github.com/netdata/netdata/blob/master/src/exporting/opentsdb/README.md) +- **M3DB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **MetricFire**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **MongoDB**: [MongoDB](https://github.com/netdata/netdata/blob/master/src/exporting/mongodb/README.md) +- **New Relic**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **OpenTSDB**: [OpenTSDB](https://github.com/netdata/netdata/blob/master/src/exporting/opentsdb/README.md), [Prometheus remote + write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **PostgreSQL**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) via [PostgreSQL Prometheus Adapter](https://github.com/CrunchyData/postgresql-prometheus-adapter) -- **Prometheus**: [Prometheus scraper](https://github.com/netdata/netdata/blob/master/exporting/prometheus/README.md) -- **TimescaleDB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md), - [netdata-timescale-relay](https://github.com/netdata/netdata/blob/master/exporting/TIMESCALE.md) -- **QuasarDB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **SignalFx**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Splunk**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **TiKV**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Thanos**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **VictoriaMetrics**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) -- **Wavefront**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md) +- **Prometheus**: [Prometheus scraper](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/README.md) +- **TimescaleDB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md), + [netdata-timescale-relay](https://github.com/netdata/netdata/blob/master/src/exporting/TIMESCALE.md) +- **QuasarDB**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **SignalFx**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Splunk**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **TiKV**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Thanos**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **VictoriaMetrics**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) +- **Wavefront**: [Prometheus remote write](https://github.com/netdata/netdata/blob/master/src/exporting/prometheus/remote_write/README.md) Can't find your preferred external time-series database? Ask our [community](https://community.netdata.cloud/) for solutions, or file an [issue on diff --git a/docs/guides/using-host-labels.md b/docs/guides/using-host-labels.md index f6db314060..5abf305be1 100644 --- a/docs/guides/using-host-labels.md +++ b/docs/guides/using-host-labels.md @@ -196,7 +196,7 @@ documentation](https://github.com/netdata/netdata/blob/master/health/REFERENCE.m ### Host labels in metrics exporting -If you have enabled any metrics exporting via our experimental [exporters](https://github.com/netdata/netdata/blob/master/exporting/README.md), any new host +If you have enabled any metrics exporting via our experimental [exporters](https://github.com/netdata/netdata/blob/master/src/exporting/README.md), any new host labels you created manually are sent to the destination database alongside metrics. You can change this behavior by editing `exporting.conf`, and you can even send automatically-generated labels on with exported metrics. @@ -221,7 +221,7 @@ send automatic labels = yes ``` By applying labels to exported metrics, you can more easily parse historical metrics with the labels applied. To learn -more about exporting, read the [documentation](https://github.com/netdata/netdata/blob/master/exporting/README.md). +more about exporting, read the [documentation](https://github.com/netdata/netdata/blob/master/src/exporting/README.md). ## Metric labels diff --git a/exporting/README.md b/exporting/README.md deleted file mode 100644 index 8a52968eed..0000000000 --- a/exporting/README.md +++ /dev/null @@ -1,327 +0,0 @@ - - -# Exporting reference - -Welcome to the exporting engine reference guide. This guide contains comprehensive information about enabling, -configuring, and monitoring Netdata's exporting engine, which allows you to send metrics to external time-series -databases. - -For a quick introduction to the exporting engine's features, read our doc on [exporting metrics to time-series -databases](https://github.com/netdata/netdata/blob/master/docs/export/external-databases.md), or jump in to [enabling a connector](https://github.com/netdata/netdata/blob/master/docs/export/enable-connector.md). - -The exporting engine has a modular structure and supports metric exporting via multiple exporting connector instances at -the same time. You can have different update intervals and filters configured for every exporting connector instance. - -When you enable the exporting engine and a connector, the Netdata Agent exports metrics _beginning from the time you -restart its process_, not the entire [database of long-term metrics](https://github.com/netdata/netdata/blob/master/docs/store/change-metrics-storage.md). - -Since Netdata collects thousands of metrics per server per second, which would easily congest any database server when -several Netdata servers are sending data to it, Netdata allows sending metrics at a lower frequency, by resampling them. - -So, although Netdata collects metrics every second, it can send to the external database servers averages or sums every -X seconds (though, it can send them per second if you need it to). - -## Features - -### Integration - -The exporting engine uses a number of connectors to send Netdata metrics to external time-series databases. See our -[list of supported databases](https://github.com/netdata/netdata/blob/master/docs/export/external-databases.md#supported-databases) for information on which -connector to enable and configure for your database of choice. - -- [**AWS Kinesis Data Streams**](https://github.com/netdata/netdata/blob/master/exporting/aws_kinesis/README.md): Metrics are sent to the service in `JSON` - format. -- [**Google Cloud Pub/Sub Service**](https://github.com/netdata/netdata/blob/master/exporting/pubsub/README.md): Metrics are sent to the service in `JSON` - format. -- [**Graphite**](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md): A plaintext interface. Metrics are sent to the database server as - `prefix.hostname.chart.dimension`. `prefix` is configured below, `hostname` is the hostname of the machine (can - also be configured). Learn more in our guide to [export and visualize Netdata metrics in - Graphite](https://github.com/netdata/netdata/blob/master/exporting/graphite/README.md). -- [**JSON** document databases](https://github.com/netdata/netdata/blob/master/exporting/json/README.md) -- [**OpenTSDB**](https://github.com/netdata/netdata/blob/master/exporting/opentsdb/README.md): Use a plaintext or HTTP interfaces. Metrics are sent to - OpenTSDB as `prefix.chart.dimension` with tag `host=hostname`. -- [**MongoDB**](https://github.com/netdata/netdata/blob/master/exporting/mongodb/README.md): Metrics are sent to the database in `JSON` format. -- [**Prometheus**](https://github.com/netdata/netdata/blob/master/exporting/prometheus/README.md): Use an existing Prometheus installation to scrape metrics - from node using the Netdata API. -- [**Prometheus remote write**](https://github.com/netdata/netdata/blob/master/exporting/prometheus/remote_write/README.md). A binary snappy-compressed protocol - buffer encoding over HTTP. Supports many [storage - providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). -- [**TimescaleDB**](https://github.com/netdata/netdata/blob/master/exporting/TIMESCALE.md): Use a community-built connector that takes JSON streams from a - Netdata client and writes them to a TimescaleDB table. - -### Chart filtering - -Netdata can filter metrics, to send only a subset of the collected metrics. You can use the -configuration file - -```txt -[prometheus:exporter] - send charts matching = system.* -``` - -or the URL parameter `filter` in the `allmetrics` API call. - -```txt -http://localhost:19999/api/v1/allmetrics?format=shell&filter=system.* -``` - -### Operation modes - -Netdata supports three modes of operation for all exporting connectors: - -- `as-collected` sends to external databases the metrics as they are collected, in the units they are collected. - So, counters are sent as counters and gauges are sent as gauges, much like all data collectors do. For example, - to calculate CPU utilization in this format, you need to know how to convert kernel ticks to percentage. - -- `average` sends to external databases normalized metrics from the Netdata database. In this mode, all metrics - are sent as gauges, in the units Netdata uses. This abstracts data collection and simplifies visualization, but - you will not be able to copy and paste queries from other sources to convert units. For example, CPU utilization - percentage is calculated by Netdata, so Netdata will convert ticks to percentage and send the average percentage - to the external database. - -- `sum` or `volume`: the sum of the interpolated values shown on the Netdata graphs is sent to the external - database. So, if Netdata is configured to send data to the database every 10 seconds, the sum of the 10 values - shown on the Netdata charts will be used. - -Time-series databases suggest to collect the raw values (`as-collected`). If you plan to invest on building your -monitoring around a time-series database and you already know (or you will invest in learning) how to convert units -and normalize the metrics in Grafana or other visualization tools, we suggest to use `as-collected`. - -If, on the other hand, you just need long term archiving of Netdata metrics and you plan to mainly work with -Netdata, we suggest to use `average`. It decouples visualization from data collection, so it will generally be a lot -simpler. Furthermore, if you use `average`, the charts shown in the external service will match exactly what you -see in Netdata, which is not necessarily true for the other modes of operation. - -### Independent operation - -This code is smart enough, not to slow down Netdata, independently of the speed of the external database server. - -> ❗ You should keep in mind though that many exporting connector instances can consume a lot of CPU resources if they -> run their batches at the same time. You can set different update intervals for every exporting connector instance, -> but even in that case they can occasionally synchronize their batches for a moment. - -## Configuration - -Here are the configuration blocks for every supported connector. Your current `exporting.conf` file may look a little -different. - -You can configure each connector individually using the available [options](#options). The -`[graphite:my_graphite_instance]` block contains examples of some of these additional options in action. - -```conf -[exporting:global] - enabled = yes - send configured labels = no - send automatic labels = no - update every = 10 - -[prometheus:exporter] - send names instead of ids = yes - send configured labels = yes - send automatic labels = no - send charts matching = * - send hosts matching = localhost * - prefix = netdata - -[graphite:my_graphite_instance] - enabled = yes - destination = localhost:2003 - data source = average - prefix = Netdata - hostname = my-name - update every = 10 - buffer on failures = 10 - timeout ms = 20000 - send charts matching = * - send hosts matching = localhost * - send names instead of ids = yes - send configured labels = yes - send automatic labels = yes - -[prometheus_remote_write:my_prometheus_remote_write_instance] - enabled = yes - destination = localhost - remote write URL path = /receive - -[kinesis:my_kinesis_instance] - enabled = yes - destination = us-east-1 - stream name = netdata - aws_access_key_id = my_access_key_id - aws_secret_access_key = my_aws_secret_access_key - -[pubsub:my_pubsub_instance] - enabled = yes - destination = pubsub.googleapis.com - credentials file = /etc/netdata/pubsub_credentials.json - project id = my_project - topic id = my_topic - -[mongodb:my_mongodb_instance] - enabled = yes - destination = localhost - database = my_database - collection = my_collection - -[json:my_json_instance] - enabled = yes - destination = localhost:5448 - -[opentsdb:my_opentsdb_plaintext_instance] - enabled = yes - destination = localhost:4242 - -[opentsdb:http:my_opentsdb_http_instance] - enabled = yes - destination = localhost:4242 - username = my_username - password = my_password - -[opentsdb:https:my_opentsdb_https_instance] - enabled = yes - destination = localhost:8082 -``` - -### Sections - -- `[exporting:global]` is a section where you can set your defaults for all exporting connectors -- `[prometheus:exporter]` defines settings for Prometheus exporter API queries (e.g.: - `http://NODE:19999/api/v1/allmetrics?format=prometheus&help=yes&source=as-collected`). -- `[:]` keeps settings for a particular exporting connector instance, where: - - `type` selects the exporting connector type: graphite | opentsdb:telnet | opentsdb:http | - prometheus_remote_write | json | kinesis | pubsub | mongodb. For graphite, opentsdb, - json, and prometheus_remote_write connectors you can also use `:http` or `:https` modifiers - (e.g.: `opentsdb:https`). - - `name` can be arbitrary instance name you chose. - -### Options - -Configure individual connectors and override any global settings with the following options. - -- `enabled = yes | no`, enables or disables an exporting connector instance - -- `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames, IPs (IPv4 and IPv6) and - ports to connect to. Netdata will use the **first available** to send the metrics. - - The format of each item in this list, is: `[PROTOCOL:]IP[:PORT]`. - - `PROTOCOL` can be `udp` or `tcp`. `tcp` is the default and only supported by the current exporting engine. - - `IP` can be `XX.XX.XX.XX` (IPv4), or `[XX:XX...XX:XX]` (IPv6). For IPv6 you can to enclose the IP in `[]` to - separate it from the port. - - `PORT` can be a number of a service name. If omitted, the default port for the exporting connector will be used - (graphite = 2003, opentsdb = 4242). - - Example IPv4: - -```conf - destination = 10.11.14.2:4242 10.11.14.3:4242 10.11.14.4:4242 -``` - - Example IPv6 and IPv4 together: - -```conf - destination = [ffff:...:0001]:2003 10.11.12.1:2003 -``` - - When multiple servers are defined, Netdata will try the next one when the previous one fails. - - Netdata also ships `nc-exporting.sh`, a script that can be used as a fallback exporting connector to save the - metrics to disk and push them to the time-series database when it becomes available again. It can also be used to - monitor / trace / debug the metrics Netdata generates. - - For the Kinesis exporting connector `destination` should be set to an AWS region (for example, `us-east-1`). - - For the MongoDB exporting connector `destination` should be set to a - [MongoDB URI](https://docs.mongodb.com/manual/reference/connection-string/). - - For the Pub/Sub exporting connector `destination` can be set to a specific service endpoint. - -- `data source = as collected`, or `data source = average`, or `data source = sum`, selects the kind of data that will - be sent to the external database. - -- `hostname = my-name`, is the hostname to be used for sending data to the external database server. By default this - is `[global].hostname`. - -- `prefix = Netdata`, is the prefix to add to all metrics. - -- `update every = 10`, is the number of seconds between sending data to the external database. Netdata will add some - randomness to this number, to prevent stressing the external server when many Netdata servers send data to the same - database. This randomness does not affect the quality of the data, only the time they are sent. - -- `buffer on failures = 10`, is the number of iterations (each iteration is `update every` seconds) to buffer data, - when the external database server is not available. If the server fails to receive the data after that many - failures, data loss on the connector instance is expected (Netdata will also log it). - -- `timeout ms = 20000`, is the timeout in milliseconds to wait for the external database server to process the data. - By default this is `2 * update_every * 1000`. - -- `send hosts matching = localhost *` includes one or more space separated patterns, using `*` as wildcard (any number - of times within each pattern). The patterns are checked against the hostname (the localhost is always checked as - `localhost`), allowing us to filter which hosts will be sent to the external database when this Netdata is a central - Netdata aggregating multiple hosts. A pattern starting with `!` gives a negative match. So to match all hosts named - `*db*` except hosts containing `*child*`, use `!*child* *db*` (so, the order is important: the first - pattern matching the hostname will be used - positive or negative). - -- `send charts matching = *` includes one or more space separated patterns, using `*` as wildcard (any number of times - within each pattern). The patterns are checked against both chart id and chart name. A pattern starting with `!` - gives a negative match. So to match all charts named `apps.*` except charts ending in `*reads`, use `!*reads - apps.*` (so, the order is important: the first pattern matching the chart id or the chart name will be used - - positive or negative). There is also a URL parameter `filter` that can be used while querying `allmetrics`. The URL - parameter has a higher priority than the configuration option. - -- `send names instead of ids = yes | no` controls the metric names Netdata should send to the external database. - Netdata supports names and IDs for charts and dimensions. Usually IDs are unique identifiers as read by the system - and names are human friendly labels (also unique). Most charts and metrics have the same ID and name, but in several - cases they are different: disks with device-mapper, interrupts, QoS classes, statsd synthetic charts, etc. - -- `send configured labels = yes | no` controls if host labels defined in the `[host labels]` section in `netdata.conf` - should be sent to the external database - -- `send automatic labels = yes | no` controls if automatically created labels, like `_os_name` or `_architecture` - should be sent to the external database - -## HTTPS - -Netdata can send metrics to external databases using the TLS/SSL protocol. Unfortunately, some of -them does not support encrypted connections, so you will have to configure a reverse proxy to enable -HTTPS communication between Netdata and an external database. You can set up a reverse proxy with -[Nginx](https://github.com/netdata/netdata/blob/master/docs/Running-behind-nginx.md). - -## Exporting engine monitoring - -Netdata creates five charts in the dashboard, under the **Netdata Monitoring** section, to help you monitor the health -and performance of the exporting engine itself: - -1. **Buffered metrics**, the number of metrics Netdata added to the buffer for dispatching them to the - external database server. - -2. **Exporting data size**, the amount of data (in KB) Netdata added the buffer. - -3. **Exporting operations**, the number of operations performed by Netdata. - -4. **Exporting thread CPU usage**, the CPU resources consumed by the Netdata thread, that is responsible for sending - the metrics to the external database server. - -![image](https://cloud.githubusercontent.com/assets/2662304/20463536/eb196084-af3d-11e6-8ee5-ddbd3b4d8449.png) - -## Exporting engine alerts - -Netdata adds 3 alerts: - -1. `exporting_last_buffering`, number of seconds since the last successful buffering of exported data -2. `exporting_metrics_sent`, percentage of metrics sent to the external database server -3. `exporting_metrics_lost`, number of metrics lost due to repeating failures to contact the external database server - -![image](https://cloud.githubusercontent.com/assets/2662304/20463779/a46ed1c2-af43-11e6-91a5-07ca4533cac3.png) - - diff --git a/exporting/TIMESCALE.md b/exporting/TIMESCALE.md deleted file mode 100644 index 8ca61b75e0..0000000000 --- a/exporting/TIMESCALE.md +++ /dev/null @@ -1,71 +0,0 @@ - - -# Writing metrics to TimescaleDB - -Thanks to Netdata's community of developers and system administrators, and Mahlon Smith -([GitHub](https://github.com/mahlonsmith)/[Website](http://www.martini.nu/)) in particular, Netdata now supports -archiving metrics directly to TimescaleDB. - -What's TimescaleDB? Here's how their team defines the project on their [GitHub page](https://github.com/timescale/timescaledb): - -> TimescaleDB is an open-source database designed to make SQL scalable for time-series data. It is engineered up from -> PostgreSQL, providing automatic partitioning across time and space (partitioning key), as well as full SQL support. - -## Quickstart - -To get started archiving metrics to TimescaleDB right away, check out Mahlon's [`netdata-timescale-relay` -repository](https://github.com/mahlonsmith/netdata-timescale-relay) on GitHub. Please be aware that backends subsystem -was removed and Netdata configuration should be moved to the new `exporting.conf` configuration file. Use -```conf -[json:my_instance] -``` -in `exporting.conf` instead of -```conf -[backend] - type = json -``` -in `netdata.conf`. - -This small program takes JSON streams from a Netdata client and writes them to a PostgreSQL (aka TimescaleDB) table. -You'll run this program in parallel with Netdata, and after a short [configuration -process](https://github.com/mahlonsmith/netdata-timescale-relay#configuration), your metrics should start populating -TimescaleDB. - -Finally, another member of Netdata's community has built a project that quickly launches Netdata, TimescaleDB, and -Grafana in easy-to-manage Docker containers. Rune Juhl Jacobsen's -[project](https://github.com/runejuhl/grafana-timescaledb) uses a `Makefile` to create everything, which makes it -perfect for testing and experimentation. - -## Netdata↔TimescaleDB in action - -Aside from creating incredible contributions to Netdata, Mahlon works at [LAIKA](https://www.laika.com/), an -Oregon-based animation studio that's helped create acclaimed films like _Coraline_ and _Kubo and the Two Strings_. - -As part of his work to maintain the company's infrastructure of render farms, workstations, and virtual machines, he's -using Netdata, `netdata-timescale-relay`, and TimescaleDB to store Netdata metrics alongside other data from other -sources. - -> LAIKA is a long-time PostgreSQL user and added TimescaleDB to their infrastructure in 2018 to help manage and store -> their IT metrics and time-series data. So far, the tool has been in production at LAIKA for over a year and helps them -> with their use case of time-based logging, where they record over 8 million metrics an hour for netdata content alone. - -By archiving Netdata metrics to a database like TimescaleDB, LAIKA can consolidate metrics data from distributed -machines efficiently. Mahlon can then correlate Netdata metrics with other sources directly in TimescaleDB. - -And, because LAIKA will soon be storing years worth of Netdata metrics data in TimescaleDB, they can analyze long-term -metrics as their films move from concept to final cut. - -Read the full blog post from LAIKA at the [TimescaleDB -blog](https://blog.timescale.com/blog/writing-it-metrics-from-netdata-to-timescaledb/amp/). - -Thank you to Mahlon, Rune, TimescaleDB, and the members of the Netdata community that requested and then built this -exporting connection between Netdata and TimescaleDB! - - diff --git a/exporting/WALKTHROUGH.md b/exporting/WALKTHROUGH.md deleted file mode 100644 index 9435be1b32..0000000000 --- a/exporting/WALKTHROUGH.md +++ /dev/null @@ -1,260 +0,0 @@ -# Netdata, Prometheus, Grafana stack - -## Intro - -In this article I will walk you through the basics of getting Netdata, Prometheus and Grafana all working together and -monitoring your application servers. This article will be using docker on your local workstation. We will be working -with docker in an ad-hoc way, launching containers that run `/bin/bash` and attaching a TTY to them. I use docker here -in a purely academic fashion and do not condone running Netdata in a container. I pick this method so individuals -without cloud accounts or access to VMs can try this out and for it's speed of deployment. - -## Why Netdata, Prometheus, and Grafana - -Some time ago I was introduced to Netdata by a coworker. We were attempting to troubleshoot python code which seemed to -be bottlenecked. I was instantly i