diff options
author | Vladimir Kobal <vlad@prokk.net> | 2019-06-07 11:48:32 +0300 |
---|---|---|
committer | Chris Akritidis <43294513+cakrit@users.noreply.github.com> | 2019-06-07 10:48:32 +0200 |
commit | 77781d033da95d91e00970572e0abf95946907af (patch) | |
tree | b4cd12ab3e7a0369118f45dbe2b24db8139a5022 | |
parent | 56c502be51dc7e441751e0de23a06d7e3ff78e2b (diff) |
Prometheus remote write backend (#6062)
* Add Prometheus remote write backend prototype
* Fix autotools issues
* Send HTTP POST request
* Add parameters to HTTP header
* Discard HTTP responce 200
* Update CMake build configuration
* Fix Codacy issue
* Check for C++ binary
* Fix compilation without remote write backend
* Add options to the installer script
* Fix configure script warning
* Fix make dist
* Downgrade to ByteSize for better compatibility
* Integrate remote write more tightly into the existing backends code
* Cleanup
* Fix build error
* Parse host tags
* Fix Codacy issue
* Fix counters for buffered data
* Rename preprocessor symbol
* Better error handling
* Cleanup
* Update the documentation
-rw-r--r-- | .gitignore | 4 | ||||
-rw-r--r-- | CMakeLists.txt | 51 | ||||
-rw-r--r-- | Makefile.am | 24 | ||||
-rw-r--r-- | backends/README.md | 8 | ||||
-rw-r--r-- | backends/backends.c | 144 | ||||
-rw-r--r-- | backends/backends.h | 4 | ||||
-rw-r--r-- | backends/prometheus/Makefile.am | 4 | ||||
-rw-r--r-- | backends/prometheus/backend_prometheus.c | 198 | ||||
-rw-r--r-- | backends/prometheus/backend_prometheus.h | 15 | ||||
-rw-r--r-- | backends/prometheus/remote_write/Makefile.am | 14 | ||||
-rw-r--r-- | backends/prometheus/remote_write/README.md | 30 | ||||
-rw-r--r-- | backends/prometheus/remote_write/remote_write.cc | 117 | ||||
-rw-r--r-- | backends/prometheus/remote_write/remote_write.h | 30 | ||||
-rw-r--r-- | backends/prometheus/remote_write/remote_write.proto | 29 | ||||
-rw-r--r-- | configure.ac | 73 | ||||
-rw-r--r-- | database/rrd.h | 2 | ||||
-rw-r--r-- | database/rrddim.c | 6 | ||||
-rw-r--r-- | database/rrdset.c | 2 | ||||
-rw-r--r-- | libnetdata/socket/socket.c | 6 | ||||
-rwxr-xr-x | netdata-installer.sh | 5 |
20 files changed, 732 insertions, 34 deletions
diff --git a/.gitignore b/.gitignore index 1d9cd173d5..f393adbfd7 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,10 @@ xenstat.plugin cgroup-network !cgroup-network/ +# protoc generated files +*.pb.cc +*.pb.h + # installation artifacts packaging/installer/.environment.sh *.tar.* diff --git a/CMakeLists.txt b/CMakeLists.txt index e774bead93..690c1272bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,6 +241,27 @@ find_library(HAVE_KINESIS aws-cpp-sdk-kinesis) # later we use: # ${HAVE_KINESIS} + +# ----------------------------------------------------------------------------- +# Detect libprotobuf + +pkg_check_modules(PROTOBUF protobuf) +# later we use: +# ${PROTOBUF_LIBRARIES} +# ${PROTOBUF_CFLAGS_OTHER} +# ${PROTOBUF_INCLUDE_DIRS} + + +# ----------------------------------------------------------------------------- +# Detect libsnappy + +pkg_check_modules(SNAPPY snappy) +# later we use: +# ${SNAPPY_LIBRARIES} +# ${SNAPPY_CFLAGS_OTHER} +# ${SNAPPY_INCLUDE_DIRS} + + # ----------------------------------------------------------------------------- # netdata files @@ -547,6 +568,11 @@ set(KINESIS_BACKEND_FILES backends/aws_kinesis/aws_kinesis_put_record.h ) +set(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES + backends/prometheus/remote_write/remote_write.cc + backends/prometheus/remote_write/remote_write.h + ) + set(DAEMON_FILES daemon/common.c daemon/common.h @@ -612,6 +638,29 @@ ELSE() ENDIF() # ----------------------------------------------------------------------------- +# prometheus remote write backend + +IF(PROTOBUF_LIBRARIES AND SNAPPY_LIBRARIES) + SET(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE True) +ELSE() + SET(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE False) +ENDIF() + +IF(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE) + message(STATUS "prometheus remote write backend: enabled") + + find_package(Protobuf REQUIRED) + protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS backends/prometheus/remote_write/remote_write.proto) + + list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROTO_SRCS} ${PROTO_HDRS}) + list(APPEND NETDATA_COMMON_LIBRARIES ${PROTOBUF_LIBRARIES} ${SNAPPY_LIBRARIES}) + list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR}) + list(APPEND NETDATA_COMMON_CFLAGS ${PROTOBUF_CFLAGS_OTHER} ${SNAPPY_CFLAGS_OTHER}) +ELSE() + message(STATUS "prometheus remote write backend: disabled (requires protobuf and snappy libraries)") +ENDIF() + +# ----------------------------------------------------------------------------- # netdata set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} m ${CMAKE_THREAD_LIBS_INIT}) @@ -648,7 +697,7 @@ ELSEIF(MACOS) ENDIF() -IF(ENABLE_BACKEND_KINESIS) +IF(ENABLE_BACKEND_KINESIS OR ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE) set_property(TARGET netdata PROPERTY CXX_STANDARD 11) set_property(TARGET netdata PROPERTY CMAKE_CXX_STANDARD_REQUIRED ON) ENDIF() diff --git a/Makefile.am b/Makefile.am index d460fc7679..6034580e6b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -441,6 +441,12 @@ KINESIS_BACKEND_FILES = \ backends/aws_kinesis/aws_kinesis_put_record.h \ $(NULL) +PROMETHEUS_REMOTE_WRITE_BACKEND_FILES = \ + backends/prometheus/remote_write/remote_write.cc \ + backends/prometheus/remote_write/remote_write.h \ + backends/prometheus/remote_write/remote_write.proto \ + $(NULL) + DAEMON_FILES = \ daemon/common.c \ daemon/common.h \ @@ -505,14 +511,13 @@ NETDATA_COMMON_LIBS = \ $(OPTIONAL_JUDY_LIBS) \ $(OPTIONAL_SSL_LIBS) \ $(NULL) -# TODO: Find more graceful way to add libs for AWS Kinesis sbin_PROGRAMS += netdata netdata_SOURCES = $(NETDATA_FILES) netdata_LDADD = \ $(NETDATA_COMMON_LIBS) \ $(NULL) -if ENABLE_BACKEND_KINESIS +if ENABLE_CXX_LINKER netdata_LINK = $(CXXLD) $(CXXFLAGS) $(LDFLAGS) -o $@ else netdata_LINK = $(CCLD) $(CFLAGS) $(LDFLAGS) -o $@ @@ -575,3 +580,18 @@ if ENABLE_BACKEND_KINESIS netdata_SOURCES += $(KINESIS_BACKEND_FILES) netdata_LDADD += $(OPTIONAL_KINESIS_LIBS) endif + +if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE + netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES) + netdata_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS) + BUILT_SOURCES = \ + backends/prometheus/remote_write/remote_write.pb.cc \ + backends/prometheus/remote_write/remote_write.pb.h \ + $(NULL) + nodist_netdata_SOURCES = $(BUILT_SOURCES) + +backends/prometheus/remote_write/remote_write.pb.cc \ +backends/prometheus/remote_write/remote_write.pb.h: backends/prometheus/remote_write/remote_write.proto + $(PROTOC) --proto_path=$(srcdir) --cpp_out=$(builddir) $^ + +endif diff --git a/backends/README.md b/backends/README.md index efaba0caac..7218ff6ed7 100644 --- a/backends/README.md +++ b/backends/README.md @@ -32,6 +32,12 @@ X seconds (though, it can send them per second if you need it to). - **prometheus** is described at [prometheus page](prometheus/) since it pulls data from netdata. + - **prometheus remote write** (a binary snappy-compressed protocol buffer encoding over HTTP used by + a lot of [storage providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage)) + + metrics are labeled in the format, which is used by Netdata for the [plaintext prometheus protocol](prometheus/). + Notes on using the remote write backend are [here](prometheus/remote_write/). + - **AWS Kinesis Data Streams** metrics are sent to the service in `JSON` format. @@ -70,7 +76,7 @@ of `netdata.conf` from your netdata): ``` [backend] enabled = yes | no - type = graphite | opentsdb | json | kinesis + type = graphite | opentsdb | json | prometheus_remote_write | kinesis host tags = list of TAG=VALUE destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used, or a region for kinesis data source = average | sum | as collected diff --git a/backends/backends.c b/backends/backends.c index 0e79189162..7108a2a87d 100644 --- a/backends/backends.c +++ b/backends/backends.c @@ -260,6 +260,12 @@ void *backends_main(void *ptr) { char *kinesis_auth_key_id = NULL, *kinesis_secure_key = NULL, *kinesis_stream_name = NULL; #endif +#if ENABLE_PROMETHEUS_REMOTE_WRITE + int do_prometheus_remote_write = 0; + BUFFER *http_request_header = buffer_create(1); +#endif + + // ------------------------------------------------------------------------ // collect configuration options @@ -285,6 +291,10 @@ void *backends_main(void *ptr) { charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT); hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT); +#if ENABLE_PROMETHEUS_REMOTE_WRITE + const char *remote_write_path = config_get(CONFIG_SECTION_BACKEND, "remote write URL path", "/receive"); +#endif + // ------------------------------------------------------------------------ // validate configuration options // and prepare for sending data to our backend @@ -337,9 +347,8 @@ void *backends_main(void *ptr) { backend_request_formatter = format_dimension_stored_json_plaintext; } -#if HAVE_KINESIS else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) { - +#if HAVE_KINESIS do_kinesis = 1; if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) { @@ -354,15 +363,31 @@ void *backends_main(void *ptr) { backend_request_formatter = format_dimension_collected_json_plaintext; else backend_request_formatter = format_dimension_stored_json_plaintext; +#else + error("AWS Kinesis support isn't compiled"); +#endif /* HAVE_KINESIS */ + } + else if (!strcmp(type, "prometheus_remote_write")) { +#if ENABLE_PROMETHEUS_REMOTE_WRITE + do_prometheus_remote_write = 1; + backend_response_checker = process_prometheus_remote_write_response; + + init_write_request(); +#else + error("Prometheus remote write support isn't compiled"); +#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */ } -#endif /* HAVE_KINESIS */ else { error("BACKEND: Unknown backend type '%s'", type); goto cleanup; } +#if ENABLE_PROMETHEUS_REMOTE_WRITE + if((backend_request_formatter == NULL && !do_prometheus_remote_write) || backend_response_checker == NULL) { +#else if(backend_request_formatter == NULL || backend_response_checker == NULL) { +#endif error("BACKEND: backend is misconfigured - disabling it."); goto cleanup; } @@ -451,6 +476,9 @@ void *backends_main(void *ptr) { size_t count_charts_total = 0; size_t count_dims_total = 0; +#if ENABLE_PROMETHEUS_REMOTE_WRITE + clear_write_request(); +#endif rrd_rdlock(); RRDHOST *host; rrdhost_foreach_read(host) { @@ -478,26 +506,45 @@ void *backends_main(void *ptr) { const char *__hostname = (host == localhost)?hostname:host->hostname; - RRDSET *st; - rrdset_foreach_read(st, host) { - if(likely(backends_can_send_rrdset(global_backend_options, st))) { - rrdset_rdlock(st); - - count_charts++; - - RRDDIM *rd; - rrddim_foreach_read(rd, st) { - if (likely(rd->last_collected_time.tv_sec >= after)) { - chart_buffered_metrics += backend_request_formatter(b, global_backend_prefix, host, __hostname, st, rd, after, before, global_backend_options); - count_dims++; - } - else { - debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before); - count_dims_skipped++; +#if ENABLE_PROMETHEUS_REMOTE_WRITE + if(do_prometheus_remote_write) { + rrd_stats_remote_write_allmetrics_prometheus( + host + , __hostname + , global_backend_prefix + , global_backend_options + , after + , before + , &count_charts + , &count_dims + , &count_dims_skipped + ); + chart_buffered_metrics += count_dims; + } + else +#endif + { + RRDSET *st; + rrdset_foreach_read(st, host) { + if(likely(backends_can_send_rrdset(global_backend_options, st))) { + rrdset_rdlock(st); + + count_charts++; + + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if (likely(rd->last_collected_time.tv_sec >= after)) { + chart_buffered_metrics += backend_request_formatter(b, global_backend_prefix, host, __hostname, st, rd, after, before, global_backend_options); + count_dims++; + } + else { + debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before); + count_dims_skipped++; + } } - } - rrdset_unlock(st); + rrdset_unlock(st); + } } } @@ -672,6 +719,43 @@ void *backends_main(void *ptr) { flags += MSG_NOSIGNAL; #endif +#if ENABLE_PROMETHEUS_REMOTE_WRITE + if(do_prometheus_remote_write) { + size_t data_size = get_write_request_size(); + + if(unlikely(!data_size)) { + error("BACKEND: write request size is out of range"); + continue; + } + + buffer_flush(b); + buffer_need_bytes(b, data_size); + if(unlikely(pack_write_request(b->buffer, &data_size))) { + error("BACKEND: cannot pack write request"); + continue; + } + b->len = data_size; + chart_buffered_bytes = (collected_number)buffer_strlen(b); + + buffer_flush(http_request_header); + buffer_sprintf(http_request_header, + "POST %s HTTP/1.1\r\n" + "Host: %s\r\n" + "Accept: */*\r\n" + "Content-Length: %zu\r\n" + "Content-Type: application/x-www-form-urlencoded\r\n\r\n", + remote_write_path, + hostname, + data_size + ); + + len = buffer_strlen(http_request_header); + send(sock, buffer_tostring(http_request_header), len, flags); + + len = data_size; + } +#endif + ssize_t written = send(sock, buffer_tostring(b), len, flags); // chart_backend_latency += now_monotonic_usec() - start_ut; if(written != -1 && (size_t)written == len) { @@ -711,6 +795,16 @@ void *backends_main(void *ptr) { } } + +#if ENABLE_PROMETHEUS_REMOTE_WRITE + if(failures) { + (void) buffer_on_failures; + failures = 0; + chart_lost_bytes = chart_buffered_bytes = get_write_request_size(); // estimated write request size + chart_data_lost_events++; + chart_lost_metrics = chart_buffered_metrics; + } +#else if(failures > buffer_on_failures) { // too bad! we are going to lose data chart_lost_bytes += buffer_strlen(b); @@ -720,6 +814,7 @@ void *backends_main(void *ptr) { chart_data_lost_events++; chart_lost_metrics = chart_buffered_metrics; } +#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */ if(unlikely(netdata_exit)) break; @@ -775,6 +870,13 @@ cleanup: } #endif +#if ENABLE_PROMETHEUS_REMOTE_WRITE + if(do_prometheus_remote_write) { + buffer_free(http_request_header); + protocol_buffers_shutdown(); + } +#endif + if(sock != -1) close(sock); diff --git a/backends/backends.h b/backends/backends.h index 1186654960..69a60e33de 100644 --- a/backends/backends.h +++ b/backends/backends.h @@ -53,4 +53,8 @@ extern int discard_response(BUFFER *b, const char *backend); #include "backends/aws_kinesis/aws_kinesis.h" #endif +#if ENABLE_PROMETHEUS_REMOTE_WRITE +#include "backends/prometheus/remote_write/remote_write.h" +#endif + #endif /* NETDATA_BACKENDS_H */ diff --git a/backends/prometheus/Makefile.am b/backends/prometheus/Makefile.am index 19554bed8e..e5f74851a3 100644 --- a/backends/prometheus/Makefile.am +++ b/backends/prometheus/Makefile.am @@ -3,6 +3,10 @@ AUTOMAKE_OPTIONS = subdir-objects MAINTAINERCLEANFILES = $(srcdir)/Makefile.in +SUBDIRS = \ + remote_write \ + $(NULL) + dist_noinst_DATA = \ README.md \ $(NULL) diff --git a/backends/prometheus/backend_prometheus.c b/backends/prometheus/backend_prometheus.c index 3641b07c5a..67342ea7ad 100644 --- a/backends/prometheus/backend_prometheus.c +++ b/backends/prometheus/backend_prometheus.c @@ -153,6 +153,8 @@ static inline char *prometheus_units_copy(char *d, const char *s, size_t usable, #define PROMETHEUS_LABELS_MAX 1024 #define PROMETHEUS_VARIABLE_MAX 256 +#define PROMETHEUS_LABELS_MAX_NUMBER 128 + struct host_variables_callback_options { RRDHOST *host; BUFFER *wb; @@ -307,7 +309,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED); int homogeneous = 1; if(as_collected) { - if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK)) + if(rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK)) rrdset_update_heterogeneous_flag(st); if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) @@ -537,6 +539,177 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER rrdhost_unlock(host); } +#if ENABLE_PROMETHEUS_REMOTE_WRITE +inline static void remote_write_split_words(char *str, char **words, int max_words) { + char *s = str; + int i = 0; + + while(*s && i < max_words - 1) { + while(*s && isspace(*s)) s++; // skip spaces to the begining of a tag name + + if(*s) + words[i] = s; + + while(*s && !isspace(*s) && *s != '=') s++; // find the end of the tag name + + if(*s != '=') { + words[i] = NULL; + break; + } + *s = '\0'; + s++; + i++; + + while(*s && isspace(*s)) s++; // skip spaces to the begining of a tag value + + if(*s && *s == '"') s++; // strip an opening quote + if(*s) + words[i] = s; + + while(*s && !isspace(*s) && *s != ',') s++; // find the end of the tag value + + if(*s && *s != ',') { + words[i] = NULL; + break; + } + if(s != words[i] && *(s - 1) == '"') *(s - 1) = '\0'; // strip a closing quote + if(*s != '\0') { + *s = '\0'; + s++; + i++; + } + } +} + +void rrd_stats_remote_write_allmetrics_prometheus( + RRDHOST *host + , const char *__hostname + , const char *prefix + , BACKEND_OPTIONS backend_options + , time_t after + , time_t before + , size_t *count_charts + , size_t *count_dims + , size_t *count_dims_skipped +) { + char hostname[PROMETHEUS_ELEMENT_MAX + 1]; + prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX); + + add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS); + + if(host->tags && *(host->tags)) { + char tags[PROMETHEUS_LABELS_MAX + 1]; + strncpy(tags, host->tags, PROMETHEUS_LABELS_MAX); + char *words[PROMETHEUS_LABELS_MAX_NUMBER] = {NULL}; + int i; + + remote_write_split_words(tags, words, PROMETHEUS_LABELS_MAX_NUMBER); + + add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS); + + for(i = 0; words[i] != NULL && words[i + 1] != NULL && (i + 1) < PROMETHEUS_LABELS_MAX_NUMBER; i += 2) { + add_tag(words[i], words[i + 1]); + } + } + + // for each chart + RRDSET *st; + rrdset_foreach_read(st, host) { + char chart[PROMETHEUS_ELEMENT_MAX + 1]; + char context[PROMETHEUS_ELEMENT_MAX + 1]; + char family[PROMETHEUS_ELEMENT_MAX + 1]; + char units[PROMETHEUS_ELEMENT_MAX + 1] = ""; + + prometheus_label_copy(chart, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX); + prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX); + + if(likely(backends_can_send_rrdset(backend_options, st))) { + rrdset_rdlock(st); + + (*count_charts)++; + + int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED); + int homogeneous = 1; + if(as_collected) { + if(rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK)) + rrdset_update_heterogeneous_flag(st); + + if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) + homogeneous = 0; + } + else { + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0); + } + + // for each dimension + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { + char name[PROMETHEUS_LABELS_MAX + 1]; + char dimension[PROMETHEUS_ELEMENT_MAX + 1]; + char *suffix = ""; + + if (as_collected) { + // we need as-collected / raw data + + if(unlikely(rd->last_collected_time.tv_sec < after)) { + debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before); + (*count_dims_skipped)++; + continue; + } + + if(homogeneous) { + // all the dimensions of the chart, has the same algorithm, multiplier and divisor + // we add all dimensions as labels + + prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", prefix, context, suffix); + + add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)); + (*count_dims)++; + } + else { + // the dimensions of the chart, do not have the same algorithm, multiplier or divisor + // we create a metric per dimension + + prometheus_name_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", prefix, context, dimension, suffix); + + add_metric(name, chart, family, NULL, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)); + (*count_dims)++; + } + } + else { + // we need average or sum of the data + + time_t first_t = after, last_t = before; + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t); + + if(!isnan(value) && !isinf(value)) { + + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE) + suffix = "_average"; + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM) + suffix = "_sum"; + + prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX); + snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", prefix, context, units, suffix); + + add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time)); + (*count_dims)++; + } + } + } + } + + rrdset_unlock(st); + } + } +} +#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */ + static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, PROMETHEUS_OUTPUT_OPTIONS output_options) { if(!server || !*server) server = "default"; @@ -599,3 +772,26 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFF } rrd_unlock(); } + +#if ENABLE_PROMETHEUS_REMOTE_WRITE +int process_prometheus_remote_write_response(BUFFER *b) { + if(unlikely(!b)) return 1; + + const char *s = buffer_tostring(b); + int len = buffer_strlen(b); + + // do nothing with HTTP response 200 + + while(!isspace(*s) && len) { + s++; + len--; + } + s++; + len--; + + if(likely(len > 4 && !strncmp(s, "200 ", 4))) + return 0; + else + return discard_response(b, "prometheus remote write"); +} +#endif diff --git a/backends/prometheus/backend_prometheus.h b/backends/prometheus/backend_prometheus.h index 72b65a22ba..d58d24004b 100644 --- a/backends/prometheus/backend_prometheus.h +++ b/backends/prometheus/backend_prometheus.h @@ -19,4 +19,19 @@ typedef enum prometheus_output_flags { extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options); extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options); +#if ENABLE_PROMETHEUS_REMOTE_WRITE +extern void rrd_stats_remote_write_allmetrics_prometheus( + RRDHOST *host + , const char *__hostname + , const char *prefix + , BACKEND_OPTIONS backend_options + , time_t after + , time_t before + , size_t *count_charts + , size_t *count_dims + , size_t *count_dims_skipped +); +extern int process_prometheus_remote_write_response(BUFFER *b); +#endif + #endif //NETDATA_BACKEND_PROMETHEUS_H diff --git a/backends/prometheus/remote_write/Makefile.am b/backends/prometheus/remote_write/Makefile.am new file mode 100644 index 0000000000..5f8f9d4c41 --- /dev/null +++ b/backends/prometheus/remote_write/Makefile.am @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +CLEANFILES = \ + remote_write.pb.cc \ + remote_write.pb.h \ + $(NULL) + +dist_noinst_DATA = \ + remote_write.proto \ + README.md \ + $(NULL) diff --git a/backends/prometheus/remote_write/README.md b/backends/prometheus/remote_write/README.md new file mode 100644 index 0000000000..73cb1daf5a --- /dev/null +++ b/backends/prometheus/remote_write/README.md @@ -0,0 +1,30 @@ +# Prometheus remote write backend + +## Prerequisites + +To use the prometheus remote write API with [storage providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage) [protobuf](https://developers.google.com/protocol-buffers/) and [snappy](https://github.com/google/snappy) libraries should be installed first. Next, netdata should be re-installed from the source. The installer will detect that the required libraries and utilities are now available. + +## Configuration + +An additional option in the backend configuration section is available for the remote write backend: + +``` +[backend] + remote write URL path = /receive +``` + +The default value is `/receive`. `remote write URL path` is used to set an endpoint path for the remote write protocol. For example, if your endpoint is `http://example.domain:example_port/storage/read` you should set + +``` +[backend] + destination = example.domain:example_port + remote write URL path = /storage/read +``` + +`buffered` and `lost` dimensions in the Netdata Backend Data Size operation monitoring chart estimate uncompressed buffer size on failures. + +## Notes + +The remote write backend does not support `buffer on failures` + +[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fbackends%2Fprometheus%2Fremote_write%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]() diff --git a/backends/prometheus/remote_write/remote_write.cc b/backends/prometheus/remote_write/re |