diff options
44 files changed, 3832 insertions, 47 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 9eaa015838..9ff64b1c38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,7 @@ IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug") set(CXX_FORMAT_SECURITY "-Werror=format-security") set(CXX_STACK_PROTECTOR "-fstack-protector-all") set(CXX_FLAGS_DEBUG "-O0") + set(CMAKE_C_STANDARD 99) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1 ${CXX_FORMAT_SIGNEDNESS} ${CXX_FORMAT_SECURITY} ${CXX_STACK_PROTECTOR} ${CXX_FLAGS_DEBUG}") ELSE() message(STATUS "building for: release") @@ -600,6 +601,23 @@ set(BACKENDS_PLUGIN_FILES backends/prometheus/backend_prometheus.h ) +set(EXPORTING_ENGINE_FILES + exporting/exporting_engine.c + exporting/exporting_engine.h + exporting/graphite/graphite.c + exporting/graphite/graphite.h + exporting/json/json.c + exporting/json/json.h + exporting/opentsdb/opentsdb.c + exporting/opentsdb/opentsdb.h + exporting/read_config.c + exporting/init_connectors.c + exporting/process_data.c + exporting/check_filters.c + exporting/send_data.c + exporting/send_internal_metrics.c + ) + set(KINESIS_BACKEND_FILES backends/aws_kinesis/aws_kinesis.c backends/aws_kinesis/aws_kinesis.h @@ -639,6 +657,7 @@ set(NETDATA_FILES ${DAEMON_FILES} ${API_PLUGIN_FILES} ${BACKENDS_PLUGIN_FILES} + ${EXPORTING_ENGINE_FILES} ${CHECKS_PLUGIN_FILES} ${HEALTH_PLUGIN_FILES} ${IDLEJITTER_PLUGIN_FILES} @@ -916,6 +935,48 @@ if(BUILD_TESTING) target_link_libraries(storage_number_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES}) add_test(NAME test_storage_number COMMAND storage_number_testdriver) + set(EXPORTING_ENGINE_TEST_FILES + exporting/tests/test_exporting_engine.c + exporting/tests/test_exporting_engine.h + exporting/tests/exporting_fixtures.c + exporting/tests/exporting_doubles.c + exporting/tests/netdata_doubles.c + exporting/tests/system_doubles.c + ) + set(TEST_NAME exporting_engine) + add_executable(${TEST_NAME}_testdriver ${EXPORTING_ENGINE_TEST_FILES} ${EXPORTING_ENGINE_FILES}) + target_compile_options( + ${TEST_NAME}_testdriver + PRIVATE + -DUNIT_TESTING + ) + target_link_options( + ${TEST_NAME}_testdriver + PRIVATE + -Wl,--wrap=read_exporting_config + -Wl,--wrap=init_connectors + -Wl,--wrap=mark_scheduled_instances + -Wl,--wrap=rrdhost_is_exportable + -Wl,--wrap=rrdset_is_exportable + -Wl,--wrap=exporting_calculate_value_from_stored_data + -Wl,--wrap=prepare_buffers + -Wl,--wrap=notify_workers + -Wl,--wrap=send_internal_metrics + -Wl,--wrap=now_realtime_sec + -Wl,--wrap=uv_thread_create + -Wl,--wrap=uv_mutex_lock + -Wl,--wrap=uv_mutex_unlock + -Wl,--wrap=uv_cond_signal + -Wl,--wrap=uv_cond_wait + -Wl,--wrap=strdupz + -Wl,--wrap=info_int + -Wl,--wrap=recv + -Wl,--wrap=send + -Wl,--wrap=connect_to_one_of + ) + target_link_libraries(${TEST_NAME}_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES}) + add_test(NAME test_${TEST_NAME} COMMAND ${TEST_NAME}_testdriver) + set(WEB_API_TEST_FILES web/api/tests/web_api.c web/server/web_client.c @@ -969,11 +1030,11 @@ if(BUILD_TESTING) set_target_properties( str2ld_testdriver storage_number_testdriver + exporting_engine_testdriver web_api_testdriver valid_urls_testdriver PROPERTIES RUNTIME_OUTPUT_DIRECTORY tests ) - endif() endif() diff --git a/Makefile.am b/Makefile.am index accf2ac691..1158cf073d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -93,6 +93,7 @@ SUBDIRS += \ collectors \ daemon \ database \ + exporting \ health \ libnetdata \ registry \ @@ -453,6 +454,23 @@ BACKENDS_PLUGIN_FILES = \ backends/prometheus/backend_prometheus.h \ $(NULL) +EXPORTING_ENGINE_FILES = \ + exporting/exporting_engine.c \ + exporting/exporting_engine.h \ + exporting/graphite/graphite.c \ + exporting/graphite/graphite.h \ + exporting/json/json.c \ + exporting/json/json.h \ + exporting/opentsdb/opentsdb.c \ + exporting/opentsdb/opentsdb.h \ + exporting/read_config.c \ + exporting/init_connectors.c \ + exporting/process_data.c \ + exporting/check_filters.c \ + exporting/send_data.c \ + exporting/send_internal_metrics.c \ + $(NULL) + KINESIS_BACKEND_FILES = \ backends/aws_kinesis/aws_kinesis.c \ backends/aws_kinesis/aws_kinesis.h \ @@ -527,6 +545,12 @@ if LINUX endif +if ENABLE_EXPORTING + NETDATA_FILES += \ + $(EXPORTING_ENGINE_FILES) \ + $(NULL) +endif + NETDATA_COMMON_LIBS = \ $(OPTIONAL_MATH_LIBS) \ $(OPTIONAL_ZLIB_LIBS) \ @@ -666,6 +690,7 @@ if ENABLE_UNITTESTS check_PROGRAMS = \ libnetdata/tests/str2ld_testdriver \ libnetdata/storage_number/tests/storage_number_testdriver \ + exporting/tests/exporting_engine_testdriver \ web/api/tests/web_api_testdriver \ web/api/tests/valid_urls_testdriver \ $(NULL) @@ -729,4 +754,46 @@ if ENABLE_UNITTESTS $(LIBNETDATA_FILES) \ $(NULL) libnetdata_storage_number_tests_storage_number_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS) + + EXPORTING_ENGINE_TEST_FILES = \ + exporting/tests/test_exporting_engine.c \ + exporting/tests/test_exporting_engine.h \ + exporting/tests/exporting_fixtures.c \ + exporting/tests/exporting_doubles.c \ + exporting/tests/netdata_doubles.c \ + exporting/tests/system_doubles.c \ + $(NULL) + exporting_tests_exporting_engine_testdriver_SOURCES = \ + $(EXPORTING_ENGINE_TEST_FILES) \ + $(EXPORTING_ENGINE_FILES) \ + $(LIBNETDATA_FILES) \ + $(NULL) + exporting_tests_exporting_engine_testdriver_CFLAGS = \ + $(AM_CFLAGS) \ + -DUNIT_TESTING \ + $(NULL) + exporting_tests_exporting_engine_testdriver_LDFLAGS = \ + -Wl,--wrap=read_exporting_config \ + -Wl,--wrap=init_connectors \ + -Wl,--wrap=mark_scheduled_instances \ + -Wl,--wrap=rrdhost_is_exportable \ + -Wl,--wrap=rrdset_is_exportable \ + -Wl,--wrap=exporting_calculate_value_from_stored_data \ + -Wl,--wrap=prepare_buffers \ + -Wl,--wrap=notify_workers \ + -Wl,--wrap=send_internal_metrics \ + -Wl,--wrap=now_realtime_sec \ + -Wl,--wrap=uv_thread_create \ + -Wl,--wrap=uv_mutex_lock \ + -Wl,--wrap=uv_mutex_unlock \ + -Wl,--wrap=uv_cond_signal \ + -Wl,--wrap=uv_cond_wait \ + -Wl,--wrap=strdupz \ + -Wl,--wrap=info_int \ + -Wl,--wrap=recv \ + -Wl,--wrap=send \ + -Wl,--wrap=connect_to_one_of \ + $(TEST_LDFLAGS) \ + $(NULL) + exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS) endif diff --git a/backends/backends.c b/backends/backends.c index 120c6e7033..9ade868822 100644 --- a/backends/backends.c +++ b/backends/backends.c @@ -269,9 +269,9 @@ void backend_set_kinesis_variables(int *default_port, #if HAVE_KINESIS *brc = process_json_response; if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_json_plaintext; + *brf = backends_format_dimension_collected_json_plaintext; else - *brf = format_dimension_stored_json_plaintext; + *brf = backends_format_dimension_stored_json_plaintext; #endif } @@ -321,9 +321,9 @@ void backend_set_mongodb_variables(int *default_port, #if HAVE_MONGOC *brc = process_json_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_json_plaintext; + *brf = backends_format_dimension_collected_json_plaintext; else - *brf = format_dimension_stored_json_plaintext; + *brf = backends_format_dimension_stored_json_plaintext; #endif } @@ -344,9 +344,9 @@ void backend_set_json_variables(int *default_port, *brc = process_json_response; if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_json_plaintext; + *brf = backends_format_dimension_collected_json_plaintext; else - *brf = format_dimension_stored_json_plaintext; + *brf = backends_format_dimension_stored_json_plaintext; } /** @@ -366,9 +366,9 @@ void backend_set_opentsdb_http_variables(int *default_port, *brc = process_opentsdb_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_opentsdb_http; + *brf = backends_format_dimension_collected_opentsdb_http; else - *brf = format_dimension_stored_opentsdb_http; + *brf = backends_format_dimension_stored_opentsdb_http; } @@ -389,9 +389,9 @@ void backend_set_opentsdb_telnet_variables(int *default_port, *brc = process_opentsdb_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_opentsdb_telnet; + *brf = backends_format_dimension_collected_opentsdb_telnet; else - *brf = format_dimension_stored_opentsdb_telnet; + *brf = backends_format_dimension_stored_opentsdb_telnet; } /** @@ -411,9 +411,9 @@ void backend_set_graphite_variables(int *default_port, *brc = process_graphite_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_graphite_plaintext; + *brf = backends_format_dimension_collected_graphite_plaintext; else - *brf = format_dimension_stored_graphite_plaintext; + *brf = backends_format_dimension_stored_graphite_plaintext; } /** @@ -624,6 +624,9 @@ void *backends_main(void *ptr) { case BACKEND_TYPE_UNKNOWN: { break; } + default: { + break; + } } #if ENABLE_PROMETHEUS_REMOTE_WRITE diff --git a/backends/backends.h b/backends/backends.h index 9da04ccee4..b259141555 100644 --- a/backends/backends.h +++ b/backends/backends.h @@ -23,9 +23,13 @@ typedef enum backend_types { BACKEND_TYPE_JSON, // Stores the data using JSON. BACKEND_TYPE_PROMETEUS, // The user selected to use Prometheus backend BACKEND_TYPE_KINESIS, // Send message to AWS Kinesis - BACKEND_TYPE_MONGODB // Send data to MongoDB collection + BACKEND_TYPE_MONGODB, // Send data to MongoDB collection + BACKEND_TYPE_NUM // Number of backend types } BACKEND_TYPE; +#ifdef ENABLE_EXPORTING +#include "exporting/exporting_engine.h" +#endif typedef int (**backend_response_checker_t)(BUFFER *); typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS); @@ -38,6 +42,7 @@ extern BACKEND_OPTIONS global_backend_options; extern const char *global_backend_prefix; extern void *backends_main(void *ptr); +BACKEND_TYPE backend_select_type(const char *type); extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options); diff --git a/backends/graphite/graphite.c b/backends/graphite/graphite.c index 805703893b..f75a93a0fd 100644 --- a/backends/graphite/graphite.c +++ b/backends/graphite/graphite.c @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- // graphite backend -int format_dimension_collected_graphite_plaintext( +int backends_format_dimension_collected_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -42,7 +42,7 @@ int format_dimension_collected_graphite_plaintext( return 1; } -int format_dimension_stored_graphite_plaintext( +int backends_format_dimension_stored_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/graphite/graphite.h b/backends/graphite/graphite.h index b7b0930fa9..498a7fcdf9 100644 --- a/backends/graphite/graphite.h +++ b/backends/graphite/graphite.h @@ -6,7 +6,7 @@ #include "backends/backends.h" -extern int format_dimension_collected_graphite_plaintext( +extern int backends_format_dimension_collected_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -18,7 +18,7 @@ extern int format_dimension_collected_graphite_plaintext( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -extern int format_dimension_stored_graphite_plaintext( +extern int backends_format_dimension_stored_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/json/json.c b/backends/json/json.c index a53c0f1437..0c7cc738f8 100644 --- a/backends/json/json.c +++ b/backends/json/json.c @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- // json backend -int format_dimension_collected_json_plaintext( +int backends_format_dimension_collected_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -74,7 +74,7 @@ int format_dimension_collected_json_plaintext( return 1; } -int format_dimension_stored_json_plaintext( +int backends_format_dimension_stored_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/json/json.h b/backends/json/json.h index 11015652e2..78ac376095 100644 --- a/backends/json/json.h +++ b/backends/json/json.h @@ -5,7 +5,7 @@ #include "backends/backends.h" -extern int format_dimension_collected_json_plaintext( +extern int backends_format_dimension_collected_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -17,7 +17,7 @@ extern int format_dimension_collected_json_plaintext( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -extern int format_dimension_stored_json_plaintext( +extern int backends_format_dimension_stored_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/opentsdb/opentsdb.c b/backends/opentsdb/opentsdb.c index 6ee559dbf3..965b4c092f 100644 --- a/backends/opentsdb/opentsdb.c +++ b/backends/opentsdb/opentsdb.c @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- // opentsdb backend -int format_dimension_collected_opentsdb_telnet( +int backends_format_dimension_collected_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -42,7 +42,7 @@ int format_dimension_collected_opentsdb_telnet( return 1; } -int format_dimension_stored_opentsdb_telnet( +int backends_format_dimension_stored_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -103,7 +103,7 @@ static inline void opentsdb_build_message(BUFFER *b, char *message, const char * ); } -int format_dimension_collected_opentsdb_http( +int backends_format_dimension_collected_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -151,7 +151,7 @@ int format_dimension_collected_opentsdb_http( return 1; } -int format_dimension_stored_opentsdb_http( +int backends_format_dimension_stored_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/opentsdb/opentsdb.h b/backends/opentsdb/opentsdb.h index b9372d914b..87d9c5cd73 100644 --- a/backends/opentsdb/opentsdb.h +++ b/backends/opentsdb/opentsdb.h @@ -5,7 +5,7 @@ #include "backends/backends.h" -extern int format_dimension_collected_opentsdb_telnet( +extern int backends_format_dimension_collected_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -17,7 +17,7 @@ extern int format_dimension_collected_opentsdb_telnet( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -extern int format_dimension_stored_opentsdb_telnet( +extern int backends_format_dimension_stored_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -31,7 +31,7 @@ extern int format_dimension_stored_opentsdb_telnet( extern int process_opentsdb_response(BUFFER *b); -int format_dimension_collected_opentsdb_http( +int backends_format_dimension_collected_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -43,7 +43,7 @@ int format_dimension_collected_opentsdb_http( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -int format_dimension_stored_opentsdb_http( +int backends_format_dimension_stored_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/configure.ac b/configure.ac index 4ae08bffc9..6fcb3baeba 100644 --- a/configure.ac +++ b/configure.ac @@ -36,6 +36,7 @@ m4_ifdef([AM_SILENT_RULES], [ ]) AC_CANONICAL_HOST AC_PROG_CC +AC_PROG_CC_C99 AM_PROG_CC_C_O AC_PROG_CXX AC_PROG_INSTALL @@ -408,6 +409,20 @@ AC_MSG_RESULT([${enable_https}]) AM_CONDITIONAL([ENABLE_HTTPS], [test "${enable_https}" = "yes"]) # ----------------------------------------------------------------------------- +# Exporting engine +AC_MSG_CHECKING([if netdata exporting engine should be used]) +if test "${UV_LIBS}"; then + enable_exporting_engine="yes" + AC_DEFINE([ENABLE_EXPORTING], [1], [netdata exporting engine usability]) + OPTIONAL_UV_CFLAGS="${UV_CFLAGS}" + OPTIONAL_UV_LIBS="${UV_LIBS}" +else + enable_exporting_engine="no" +fi +AC_MSG_RESULT([${enable_exporting_engine}]) +AM_CONDITIONAL([ENABLE_EXPORTING], [test "${enable_exporting_engine}" = "yes"]) + +# ----------------------------------------------------------------------------- # JSON-C test "${enable_jsonc}" = "yes" -a -z "${JSONC_LIBS}" && \ AC_MSG_ERROR([JSON-C required but not found. Try installing 'libjson-c-dev' or 'json-c'.]) @@ -1209,6 +1224,11 @@ AC_CONFIG_FILES([ database/Makefile database/engine/Makefile diagrams/Makefile + exporting/Makefile + exporting/graphite/Makefile + exporting/json/Makefile + exporting/opentsdb/Makefile + exporting/tests/Makefile health/Makefile health/notifications/Makefile libnetdata/Makefile diff --git a/daemon/common.h b/daemon/common.h index df36a49d4e..c04c8ad04f 100644 --- a/daemon/common.h +++ b/daemon/common.h @@ -27,7 +27,6 @@ #define config_generate(buffer, only_changed) appconfig_generate(&netdata_config, buffer, only_changed) - // ---------------------------------------------------------------------------- // netdata include files diff --git a/daemon/main.c b/daemon/main.c index ee2a437fd0..f8c6d84d45 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -75,6 +75,9 @@ struct netdata_static_thread static_threads[] = { // common plugins for all systems {"BACKENDS", NULL, NULL, 1, NULL, NULL, backends_main}, +#ifdef ENABLE_EXPORTING + {"EXPORTING", NULL, NULL, 1, NULL, NULL, exporting_main}, +#endif {"WEB_SERVER[static1]", NULL, NULL, 0, NULL, NULL, socket_listen_main_static_threaded}, {"STREAM", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread}, @@ -1105,6 +1108,7 @@ int main(int argc, char **argv) { if(!config_loaded) load_netdata_conf(NULL, 0); + // ------------------------------------------------------------------------ // initialize netdata { diff --git a/database/rrd.h b/database/rrd.h index 8efe2adb05..c75b0deebe 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -408,6 +408,7 @@ struct rrdset { // it goes around in a round-robin fashion RRDSET_FLAGS flags; // configuration flags + RRDSET_FLAGS *exporting_flags; // array of flags for exporting connector instances int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored // netdata will interpolate values for gaps lower than this @@ -627,6 +628,7 @@ struct rrdhost { |