diff options
author | Vladimir Kobal <vlad@prokk.net> | 2019-12-12 21:41:11 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-12-12 21:41:11 +0200 |
commit | 6f270819121afb743d783f9a72786e367d1048a3 (patch) | |
tree | 6e17d6764e5c6d72bb53bbfcc35cecce4b18d325 /backends | |
parent | 7278d5bcd987fb9646da4a8a837173bae3b68459 (diff) |
Implement the main flow for the Exporting Engine (#7149)
* Add top level tests
* Add a skeleton for preparing buffers
* Initialize graphite instance
* Prepare buffers for all instances
* Add Grafite collected value formatter
* Add support for exporting.conf read and parsing
* - Use new exporting_config instead of netdata_config
* Implement Grafite worker
* Disable exporting engine compilation if libuv is not available
* Add mutex locks
- Configure connectors as connector_<type> in sections of exporting.conf
- Change exporting_select_type to check for connector_ fields
* - Override exporting_config structure if there no exporting.conf so that
look ups don't fail and we maintain backwards compatibility
* Separate fixtures in unit tests
* Test exporting_discard_responce
* Test response receiving
* Test buffer sending
* Test simple connector worker
- Instance section has the format connector:instance_name
e.g graphite:my_graphite_instance
- Connectors with : in their name e.g graphite:plaintext are reserved
So graphite:plaintext is not accepted because it would activate an
instance with name "plaintext"
It should be graphite:plaintext:instance_name
* - Enable the add_connector_instance to cleanup the internal structure
by passing NULL,not NULL arguments
* Implement configurable update interval
- Add additional check to verify instance uniqueness across connectors
* Add host and chart filters
* Add the value calculation over a database series
* Add the calculated over stored data graphite connector
* Add tests for graphite connector
* Add JSON connector
* Add tests for JSON formatting functions
* Add OpenTSDB connector
* Add tests for the OpenTSDB connector
* Add temporaty notes to the documentation
Diffstat (limited to 'backends')
-rw-r--r-- | backends/backends.c | 27 | ||||
-rw-r--r-- | backends/backends.h | 7 | ||||
-rw-r--r-- | backends/graphite/graphite.c | 4 | ||||
-rw-r--r-- | backends/graphite/graphite.h | 4 | ||||
-rw-r--r-- | backends/json/json.c | 4 | ||||
-rw-r--r-- | backends/json/json.h | 4 | ||||
-rw-r--r-- | backends/opentsdb/opentsdb.c | 8 | ||||
-rw-r--r-- | backends/opentsdb/opentsdb.h | 8 |
8 files changed, 37 insertions, 29 deletions
diff --git a/backends/backends.c b/backends/backends.c index 120c6e7033..9ade868822 100644 --- a/backends/backends.c +++ b/backends/backends.c @@ -269,9 +269,9 @@ void backend_set_kinesis_variables(int *default_port, #if HAVE_KINESIS *brc = process_json_response; if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_json_plaintext; + *brf = backends_format_dimension_collected_json_plaintext; else - *brf = format_dimension_stored_json_plaintext; + *brf = backends_format_dimension_stored_json_plaintext; #endif } @@ -321,9 +321,9 @@ void backend_set_mongodb_variables(int *default_port, #if HAVE_MONGOC *brc = process_json_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_json_plaintext; + *brf = backends_format_dimension_collected_json_plaintext; else - *brf = format_dimension_stored_json_plaintext; + *brf = backends_format_dimension_stored_json_plaintext; #endif } @@ -344,9 +344,9 @@ void backend_set_json_variables(int *default_port, *brc = process_json_response; if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_json_plaintext; + *brf = backends_format_dimension_collected_json_plaintext; else - *brf = format_dimension_stored_json_plaintext; + *brf = backends_format_dimension_stored_json_plaintext; } /** @@ -366,9 +366,9 @@ void backend_set_opentsdb_http_variables(int *default_port, *brc = process_opentsdb_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_opentsdb_http; + *brf = backends_format_dimension_collected_opentsdb_http; else - *brf = format_dimension_stored_opentsdb_http; + *brf = backends_format_dimension_stored_opentsdb_http; } @@ -389,9 +389,9 @@ void backend_set_opentsdb_telnet_variables(int *default_port, *brc = process_opentsdb_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_opentsdb_telnet; + *brf = backends_format_dimension_collected_opentsdb_telnet; else - *brf = format_dimension_stored_opentsdb_telnet; + *brf = backends_format_dimension_stored_opentsdb_telnet; } /** @@ -411,9 +411,9 @@ void backend_set_graphite_variables(int *default_port, *brc = process_graphite_response; if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) - *brf = format_dimension_collected_graphite_plaintext; + *brf = backends_format_dimension_collected_graphite_plaintext; else - *brf = format_dimension_stored_graphite_plaintext; + *brf = backends_format_dimension_stored_graphite_plaintext; } /** @@ -624,6 +624,9 @@ void *backends_main(void *ptr) { case BACKEND_TYPE_UNKNOWN: { break; } + default: { + break; + } } #if ENABLE_PROMETHEUS_REMOTE_WRITE diff --git a/backends/backends.h b/backends/backends.h index 9da04ccee4..b259141555 100644 --- a/backends/backends.h +++ b/backends/backends.h @@ -23,9 +23,13 @@ typedef enum backend_types { BACKEND_TYPE_JSON, // Stores the data using JSON. BACKEND_TYPE_PROMETEUS, // The user selected to use Prometheus backend BACKEND_TYPE_KINESIS, // Send message to AWS Kinesis - BACKEND_TYPE_MONGODB // Send data to MongoDB collection + BACKEND_TYPE_MONGODB, // Send data to MongoDB collection + BACKEND_TYPE_NUM // Number of backend types } BACKEND_TYPE; +#ifdef ENABLE_EXPORTING +#include "exporting/exporting_engine.h" +#endif typedef int (**backend_response_checker_t)(BUFFER *); typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS); @@ -38,6 +42,7 @@ extern BACKEND_OPTIONS global_backend_options; extern const char *global_backend_prefix; extern void *backends_main(void *ptr); +BACKEND_TYPE backend_select_type(const char *type); extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options); diff --git a/backends/graphite/graphite.c b/backends/graphite/graphite.c index 805703893b..f75a93a0fd 100644 --- a/backends/graphite/graphite.c +++ b/backends/graphite/graphite.c @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- // graphite backend -int format_dimension_collected_graphite_plaintext( +int backends_format_dimension_collected_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -42,7 +42,7 @@ int format_dimension_collected_graphite_plaintext( return 1; } -int format_dimension_stored_graphite_plaintext( +int backends_format_dimension_stored_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/graphite/graphite.h b/backends/graphite/graphite.h index b7b0930fa9..498a7fcdf9 100644 --- a/backends/graphite/graphite.h +++ b/backends/graphite/graphite.h @@ -6,7 +6,7 @@ #include "backends/backends.h" -extern int format_dimension_collected_graphite_plaintext( +extern int backends_format_dimension_collected_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -18,7 +18,7 @@ extern int format_dimension_collected_graphite_plaintext( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -extern int format_dimension_stored_graphite_plaintext( +extern int backends_format_dimension_stored_graphite_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/json/json.c b/backends/json/json.c index a53c0f1437..0c7cc738f8 100644 --- a/backends/json/json.c +++ b/backends/json/json.c @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- // json backend -int format_dimension_collected_json_plaintext( +int backends_format_dimension_collected_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -74,7 +74,7 @@ int format_dimension_collected_json_plaintext( return 1; } -int format_dimension_stored_json_plaintext( +int backends_format_dimension_stored_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/json/json.h b/backends/json/json.h index 11015652e2..78ac376095 100644 --- a/backends/json/json.h +++ b/backends/json/json.h @@ -5,7 +5,7 @@ #include "backends/backends.h" -extern int format_dimension_collected_json_plaintext( +extern int backends_format_dimension_collected_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -17,7 +17,7 @@ extern int format_dimension_collected_json_plaintext( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -extern int format_dimension_stored_json_plaintext( +extern int backends_format_dimension_stored_json_plaintext( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/opentsdb/opentsdb.c b/backends/opentsdb/opentsdb.c index 6ee559dbf3..965b4c092f 100644 --- a/backends/opentsdb/opentsdb.c +++ b/backends/opentsdb/opentsdb.c @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- // opentsdb backend -int format_dimension_collected_opentsdb_telnet( +int backends_format_dimension_collected_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -42,7 +42,7 @@ int format_dimension_collected_opentsdb_telnet( return 1; } -int format_dimension_stored_opentsdb_telnet( +int backends_format_dimension_stored_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -103,7 +103,7 @@ static inline void opentsdb_build_message(BUFFER *b, char *message, const char * ); } -int format_dimension_collected_opentsdb_http( +int backends_format_dimension_collected_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -151,7 +151,7 @@ int format_dimension_collected_opentsdb_http( return 1; } -int format_dimension_stored_opentsdb_http( +int backends_format_dimension_stored_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from diff --git a/backends/opentsdb/opentsdb.h b/backends/opentsdb/opentsdb.h index b9372d914b..87d9c5cd73 100644 --- a/backends/opentsdb/opentsdb.h +++ b/backends/opentsdb/opentsdb.h @@ -5,7 +5,7 @@ #include "backends/backends.h" -extern int format_dimension_collected_opentsdb_telnet( +extern int backends_format_dimension_collected_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -17,7 +17,7 @@ extern int format_dimension_collected_opentsdb_telnet( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -extern int format_dimension_stored_opentsdb_telnet( +extern int backends_format_dimension_stored_opentsdb_telnet( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -31,7 +31,7 @@ extern int format_dimension_stored_opentsdb_telnet( extern int process_opentsdb_response(BUFFER *b); -int format_dimension_collected_opentsdb_http( +int backends_format_dimension_collected_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from @@ -43,7 +43,7 @@ int format_dimension_collected_opentsdb_http( , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap ); -int format_dimension_stored_opentsdb_http( +int backends_format_dimension_stored_opentsdb_http( BUFFER *b // the buffer to write data to , const char *prefix // the prefix to use , RRDHOST *host // the host this chart comes from |