diff options
Diffstat (limited to 'libnetdata')
-rw-r--r-- | libnetdata/Makefile.am | 1 | ||||
-rw-r--r-- | libnetdata/libnetdata.c | 108 | ||||
-rw-r--r-- | libnetdata/libnetdata.h | 119 | ||||
-rw-r--r-- | libnetdata/parser/Makefile.am | 9 | ||||
-rw-r--r-- | libnetdata/parser/README.md | 28 | ||||
-rw-r--r-- | libnetdata/parser/parser.c | 277 | ||||
-rw-r--r-- | libnetdata/parser/parser.h | 105 |
7 files changed, 127 insertions, 520 deletions
diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am index 4bf7791368..b81d620ba6 100644 --- a/libnetdata/Makefile.am +++ b/libnetdata/Makefile.am @@ -20,7 +20,6 @@ SUBDIRS = \ locks \ log \ onewayalloc \ - parser \ popen \ procfile \ simple_pattern \ diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c index bd3ca7c66d..8439902393 100644 --- a/libnetdata/libnetdata.c +++ b/libnetdata/libnetdata.c @@ -1684,7 +1684,7 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c return value; } -inline int pluginsd_space(char c) { +inline int pluginsd_isspace(char c) { switch(c) { case ' ': case '\t': @@ -1698,8 +1698,7 @@ inline int pluginsd_space(char c) { } } -inline int config_isspace(char c) -{ +inline int config_isspace(char c) { switch (c) { case ' ': case '\t': @@ -1713,100 +1712,23 @@ inline int config_isspace(char c) } } -// split a text into words, respecting quotes -inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char)) -{ - char *s = str, quote = 0; - size_t i = 0; - - // skip all white space - while (unlikely(custom_isspace(*s))) - s++; - - if(unlikely(!*s)) { - words[i] = NULL; - return 0; - } - - // check for quote - if (unlikely(*s == '\'' || *s == '"')) { - quote = *s; // remember the quote - s++; // skip the quote - } - - // store the first word - words[i++] = s; - - // while we have something - while (likely(*s)) { - // if it is an escape - if (unlikely(*s == '\\' && s[1])) { - s += 2; - continue; - } - - // if it is a quote - else if (unlikely(*s == quote)) { - quote = 0; - *s = ' '; - continue; - } - - // if it is a space - else if (unlikely(quote == 0 && custom_isspace(*s))) { - // terminate the word - *s++ = '\0'; - - // skip all white space - while (likely(custom_isspace(*s))) - s++; - - // check for a quote - if (unlikely(*s == '\'' || *s == '"')) { - quote = *s; // remember the quote - s++; // skip the quote - } - - // if we reached the end, stop - if (unlikely(!*s)) - break; - - // store the next word - if (likely(i < max_words)) - words[i++] = s; - else - break; - } - - // anything else - else - s++; - } - - if (i < max_words) - words[i] = NULL; +inline int group_by_label_isspace(char c) { + if(c == ',' || c == '|') + return 1; - return i; + return 0; } -inline size_t pluginsd_split_words(char *str, char **words, size_t max_words) -{ - return quoted_strings_splitter(str, words, max_words, pluginsd_space); -} +bool isspace_map_pluginsd[256] = {}; +bool isspace_map_config[256] = {}; +bool isspace_map_group_by_label[256] = {}; -bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx) { - if (unlikely(!ptr)) - return false; - return (ptr->data[idx / 64] & (1ULL << (idx % 64))); -} - -void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) { - if (unlikely(!ptr)) - return; - if (likely(value)) - ptr->data[idx / 64] |= (1ULL << (idx % 64)); - else - ptr->data[idx / 64] &= ~(1ULL << (idx % 64)); +__attribute__((constructor)) void initialize_is_space_arrays(void) { + for(int c = 0; c < 256 ; c++) { + isspace_map_pluginsd[c] = pluginsd_isspace((char) c); + isspace_map_config[c] = config_isspace((char) c); + isspace_map_group_by_label[c] = group_by_label_isspace((char) c); + } } bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length) { diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index 9a0f021e1a..80a0026896 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -591,23 +591,129 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c // Taken from linux kernel #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)])) +#ifdef ENV32BIT +typedef struct bitmap256 { + uint32_t data[8]; +} BITMAP256; + +#define bitmap256_get_bit(ptr, idx) (((ptr)->data[(idx) >> 5] & (1ULL << ((idx) & 31))) != 0) +static inline void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) { + uint32_t mask = 1U << (idx & 31); + if (value) + ptr->data[idx >> 5] |= mask; + else + ptr->data[idx >> 5] &= ~mask; +} +#else // 64bit typedef struct bitmap256 { uint64_t data[4]; } BITMAP256; -bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx); -void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value); +#define bitmap256_get_bit(ptr, idx) ((ptr)->data[(idx) >> 6] & (1ULL << ((idx) & 63))) +static inline void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) { + uint64_t mask = 1ULL << (idx & 63); + if (value) + ptr->data[idx >> 6] |= mask; + else + ptr->data[idx >> 6] &= ~mask; +} +#endif // 64bit #define COMPRESSION_MAX_MSG_SIZE 0x4000 #define PLUGINSD_LINE_MAX (COMPRESSION_MAX_MSG_SIZE - 1024) +int pluginsd_isspace(char c); int config_isspace(char c); -int pluginsd_space(char c); +int group_by_label_isspace(char c); + +extern bool isspace_map_pluginsd[256]; +extern bool isspace_map_config[256]; +extern bool isspace_map_group_by_label[256]; + +static inline size_t quoted_strings_splitter(char *str, char **words, size_t max_words, bool *isspace_map) { + char *s = str, quote = 0; + size_t i = 0; + + // skip all white space + while (unlikely(isspace_map[(uint8_t)*s])) + s++; + + if(unlikely(!*s)) { + words[i] = NULL; + return 0; + } + + // check for quote + if (unlikely(*s == '\'' || *s == '"')) { + quote = *s; // remember the quote + s++; // skip the quote + } + + // store the first word + words[i++] = s; + + // while we have something + while (likely(*s)) { + // if it is an escape + if (unlikely(*s == '\\' && s[1])) { + s += 2; + continue; + } + + // if it is a quote + else if (unlikely(*s == quote)) { + quote = 0; + *s = ' '; + continue; + } + + // if it is a space + else if (unlikely(quote == 0 && isspace_map[(uint8_t)*s])) { + // terminate the word + *s++ = '\0'; + + // skip all white space + while (likely(isspace_map[(uint8_t)*s])) + s++; + + // check for a quote + if (unlikely(*s == '\'' || *s == '"')) { + quote = *s; // remember the quote + s++; // skip the quote + } + + // if we reached the end, stop + if (unlikely(!*s)) + break; + + // store the next word + if (likely(i < max_words)) + words[i++] = s; + else + break; + } + + // anything else + else + s++; + } + + if (likely(i < max_words)) + words[i] = NULL; + + return i; +} + +#define quoted_strings_splitter_query_group_by_label(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_group_by_label) + +#define quoted_strings_splitter_config(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_config) -size_t quoted_strings_splitter(char *str, char **words, size_t max_words, int (*custom_isspace)(char)); -size_t pluginsd_split_words(char *str, char **words, size_t max_words); +#define quoted_strings_splitter_pluginsd(str, words, max_words) \ + quoted_strings_splitter(str, words, max_words, isspace_map_pluginsd) static inline char *get_word(char **words, size_t num_words, size_t index) { - if (index >= num_words) + if (unlikely(index >= num_words)) return NULL; return words[index]; @@ -664,7 +770,6 @@ extern char *netdata_configured_host_prefix; #include "libnetdata/aral/aral.h" #include "onewayalloc/onewayalloc.h" #include "worker_utilization/worker_utilization.h" -#include "parser/parser.h" #include "yaml.h" #include "http/http_defs.h" #include "gorilla/gorilla.h" diff --git a/libnetdata/parser/Makefile.am b/libnetdata/parser/Makefile.am deleted file mode 100644 index 02fe3a314f..0000000000 --- a/libnetdata/parser/Makefile.am +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later - -AUTOMAKE_OPTIONS = subdir-objects -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - -dist_noinst_DATA = \ - README.md \ - $(NULL) - diff --git a/libnetdata/parser/README.md b/libnetdata/parser/README.md deleted file mode 100644 index 136c23c692..0000000000 --- a/libnetdata/parser/README.md +++ /dev/null @@ -1,28 +0,0 @@ -<!-- -title: "Parser" -custom_edit_url: https://github.com/netdata/netdata/blob/master/parser/README.md -sidebar_label: "Parser" -learn_status: "Published" -learn_topic_type: "References" -learn_rel_path: "Developers/Database" ---> - -# Parser - -## Introduction - -Generic parser that is used to register keywords and a corresponding function that will be executed when that -keyword is encountered in the command stream (either from plugins or via streaming) - -To use a parser do the following: - -1. Define a structure that will be used to share user state across calls (user defined `void *user`) -2. Initialize the parser using `parser_init` -3. Register keywords with their associated callback function using `parser_add_keyword` -4. Start a loop for as long there is input (or parser_action returns error) - 1. Fetch the next line using `parser_next` (if needed) - 2. Process the line using `parser_action` -5. Release the parser using `parser_destroy` -6. Release the user structure - -See examples in receiver.c / pluginsd_parser.c diff --git a/libnetdata/parser/parser.c b/libnetdata/parser/parser.c deleted file mode 100644 index 8c2b93e7fb..0000000000 --- a/libnetdata/parser/parser.c +++ /dev/null @@ -1,277 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later -#include <poll.h> -#include <stdio.h> - -#include "parser.h" -#include "collectors/plugins.d/pluginsd_parser.h" - -static inline int find_first_keyword(const char *src, char *dst, int dst_size, int (*custom_isspace)(char)) { - const char *s = src, *keyword_start; - - while (unlikely(custom_isspace(*s))) s++; - keyword_start = s; - - while (likely(*s && !custom_isspace(*s)) && dst_size > 1) { - *dst++ = *s++; - dst_size--; - } - *dst = '\0'; - return dst_size == 0 ? 0 : (int) (s - keyword_start); -} - -/* - * Initialize a parser - * user : as defined by the user, will be shared across calls - * input : main input stream (auto detect stream -- file, socket, pipe) - * buffer : This is the buffer to be used (if null a buffer of size will be allocated) - * size : buffer size either passed or will be allocated - * If the buffer is auto allocated, it will auto freed when the parser is destroyed - * - * - */ - -PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd, - PARSER_INPUT_TYPE flags, void *ssl __maybe_unused) -{ - PARSER *parser; - - parser = callocz(1, sizeof(*parser)); - parser->user = user; - parser->fd = fd; - parser->fp_input = fp_input; - parser->fp_output = fp_output; -#ifdef ENABLE_HTTPS - parser->ssl_output = ssl; -#endif - parser->flags = flags; - parser->worker_job_next_id = WORKER_PARSER_FIRST_JOB; - - spinlock_init(&parser->writer.spinlock); - return parser; -} - - -static inline PARSER_KEYWORD *parser_find_keyword(PARSER *parser, const char *command) { - uint32_t hash = parser_hash_function(command); - uint32_t slot = hash % PARSER_KEYWORDS_HASHTABLE_SIZE; - PARSER_KEYWORD *t = parser->keywords.hashtable[slot]; - - if(likely(t && strcmp(t->keyword, command) == 0)) - return t; - - return NULL; -} - -/* - * Add a keyword and the corresponding function that will be called - * Multiple functions may be added - * Input : keyword - * : callback function - * : flags - * Output: > 0 registered function number - * : 0 Error - */ - -void parser_add_keyword(PARSER *parser, char *keyword, keyword_function func) { - if(unlikely(!parser || !keyword || !*keyword || !func)) - fatal("PARSER: invalid parameters"); - - PARSER_KEYWORD *t = callocz(1, sizeof(*t)); - t->worker_job_id = parser->worker_job_next_id++; - t->keyword = strdupz(keyword); - t->func = func; - - uint32_t hash = parser_hash_function(keyword); - uint32_t slot = hash % PARSER_KEYWORDS_HASHTABLE_SIZE; - - if(unlikely(parser->keywords.hashtable[slot])) - fatal("PARSER: hashtable collision between keyword '%s' and '%s' on slot %u. " - "Change the hashtable size and / or the hashing function. " - "Run the unit test to find the optimal values.", - parser->keywords.hashtable[slot]->keyword, - t->keyword, - slot - ); - - parser->keywords.hashtable[slot] = t; - - worker_register_job_name(t->worker_job_id, t->keyword); -} - -/* - * Cleanup a previously allocated parser - */ - -void parser_destroy(PARSER *parser) -{ - if (unlikely(!parser)) - return; - - dictionary_destroy(parser->inflight.functions); - - // Remove keywords - for(size_t i = 0 ; i < PARSER_KEYWORDS_HASHTABLE_SIZE; i++) { - PARSER_KEYWORD *t = parser->keywords.hashtable[i]; - if (t) { - freez(t->keyword); - freez(t); - } - } - - freez(parser); -} - - -/* - * Fetch the next line to process - * - */ - -typedef enum { - PARSER_FGETS_RESULT_OK, - PARSER_FGETS_RESULT_TIMEOUT, - PARSER_FGETS_RESULT_ERROR, - PARSER_FGETS_RESULT_EOF, -} PARSER_FGETS_RESULT; - -static inline PARSER_FGETS_RESULT parser_fgets(char *s, int size, FILE *stream) { - errno = 0; - - struct pollfd fds[1]; - int timeout_msecs = 2 * 60 * MSEC_PER_SEC; - - fds[0].fd = fileno(stream); - fds[0].events = POLLIN; - - int ret = poll(fds, 1, timeout_msecs); - - if (ret > 0) { - /* There is data to read */ - if (fds[0].revents & POLLIN) { - char *tmp = fgets(s, size, stream); - - if(unlikely(!tmp)) { - if (feof(stream)) { - error("PARSER: read failed: end of file."); - return PARSER_FGETS_RESULT_EOF; - } - - else if (ferror(stream)) { - error("PARSER: read failed: input error."); - return PARSER_FGETS_RESULT_ERROR; - } - - error("PARSER: read failed: unknown error."); - return PARSER_FGETS_RESULT_ERROR; - } - - return PARSER_FGETS_RESULT_OK; - } - else if(fds[0].revents & POLLERR) { - error("PARSER: read failed: POLLERR."); - return PARSER_FGETS_RESULT_ERROR; - } - else if(fds[0].revents & POLLHUP) { - error("PARSER: read failed: POLLHUP."); - return PARSER_FGETS_RESULT_ERROR; - } - else if(fds[0].revents & POLLNVAL) { - error("PARSER: read failed: POLLNVAL."); - return PARSER_FGETS_RESULT_ERROR; - } - - error("PARSER: poll() returned positive number, but POLLIN|POLLERR|POLLHUP|POLLNVAL are not set."); - return PARSER_FGETS_RESULT_ERROR; - } - else if (ret == 0) { - error("PARSER: timeout while waiting for data."); - return PARSER_FGETS_RESULT_TIMEOUT; - } - - error("PARSER: poll() failed with code %d.", ret); - return PARSER_FGETS_RESULT_ERROR; -} - -int parser_next(PARSER *parser, char *buffer, size_t buffer_size) { - if(likely(parser_fgets(buffer, (int)buffer_size, (FILE *)parser->fp_input) == PARSER_FGETS_RESULT_OK)) - return 0; - - return 1; -} - -/* -* Takes an initialized parser object that has an unprocessed entry (by calling parser_next) -* and if it contains a valid keyword, it will execute all the callbacks -* -*/ - -inline int parser_action(PARSER *parser, char *input) -{ - parser->line++; - - if(unlikely(parser->flags & PARSER_DEFER_UNTIL_KEYWORD)) { - char command[PLUGINSD_LINE_MAX + 1]; - bool has_keyword = find_first_keyword(input, command, PLUGINSD_LINE_MAX, pluginsd_space); - - if(!has_keyword || strcmp(command, parser->defer.end_keyword) != 0) { - if(parser->defer.response) { - buffer_strcat(parser->defer.response, input); - if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) { - // more than 10MB of data - // a bad plugin that did not send the end_keyword - internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response)); - return 1; - } - } - return 0; - } - else { - // call the action - parser->defer.action(parser, parser->defer.action_data); - - // empty everything - parser->defer.action = NULL; - parser->defer.action_data = NULL; - parser->defer.end_keyword = NULL; - parser->defer.response = NULL; - parser->flags &= ~PARSER_DEFER_UNTIL_KEYWORD; - } - return 0; - } - - char *words[PLUGINSD_MAX_WORDS]; - size_t num_words = pluginsd_split_words(input, words, PLUGINSD_MAX_WORDS); - const char *command = get_word(words, num_words, 0); - - if(unlikely(!command)) - return 0; - - PARSER_RC rc; - PARSER_KEYWORD *t = parser_find_keyword(parser, command); - if(likely(t)) { - worker_is_busy(t->worker_job_id); - rc = (*t->func)(words, num_words, parser->user); - worker_is_idle(); - } - else - rc = PARSER_RC_ERROR; - - if(rc == PARSER_RC_ERROR) { - BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL); - for(size_t i = 0; i < num_words ;i++) { - if(i) buffer_fast_strcat(wb, " ", 1); - - buffer_fast_strcat(wb, "\"", 1); - const char *s = get_word(words, num_words, i); - buffer_strcat(wb, s?s:""); - buffer_fast_strcat(wb, "\"", 1); - } - - error("PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)", - command, parser->line, buffer_tostring(wb)); - - buffer_free(wb); - } - - return (rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP); -} diff --git a/libnetdata/parser/parser.h b/libnetdata/parser/parser.h deleted file mode 100644 index a40db90705..0000000000 --- a/libnetdata/parser/parser.h +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -#ifndef NETDATA_INCREMENTAL_PARSER_H -#define NETDATA_INCREMENTAL_PARSER_H 1 - -#include "../libnetdata.h" - -#define WORKER_PARSER_FIRST_JOB 3 - -// this has to be in-sync with the same at receiver.c -#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3) - -#define PARSER_KEYWORDS_HASHTABLE_SIZE 73 // unittest finds this magic number -//#define parser_hash_function(s) djb2_hash32(s) -//#define parser_hash_function(s) fnv1_hash32(s) -//#define parser_hash_function(s) fnv1a_hash32(s) -//#define parser_hash_function(s) larson_hash32(s) -#define parser_hash_function(s) pluginsd_parser_hash32(s) - -// PARSER return codes -typedef enum __attribute__ ((__packed__)) parser_rc { - PARSER_RC_OK, // Callback was successful, go on - PARSER_RC_STOP, // Callback says STOP - PARSER_RC_ERROR // Callback failed (abort rest of callbacks) -} PARSER_RC; - -typedef enum __attribute__ ((__packed__)) parser_input_type { - PARSER_INPUT_SPLIT = (1 << 1), - PARSER_DEFER_UNTIL_KEYWORD = (1 << 2), -} PARSER_INPUT_TYPE; - -typedef PARSER_RC (*keyword_function)(char **words, size_t num_words, void *user_data); - -typedef struct parser_keyword { - size_t worker_job_id; - char *keyword; - keyword_function func; -} PARSER_KEYWORD; - -typedef struct parser { - size_t worker_job_next_id; - uint8_t version; // Parser version - int fd; // Socket - FILE *fp_input; // Input source e.g. stream - FILE *fp_output; // Stream to send commands to plugin -#ifdef ENABLE_HTTPS - NETDATA_SSL *ssl_output; -#endif - void *user; // User defined structure to hold extra state between calls - uint32_t flags; - size_t line; - - struct { - PARSER_KEYWORD *hashtable[PARSER_KEYWORDS_HASHTABLE_SIZE]; - } keywords; - - struct { - const char *end_keyword; - BUFFER *response; - void (*action)(struct parser *parser, void *action_data); - void *action_data; - } defer; - - struct { - DICTIONARY *functions; - usec_t smaller_timeout; - } inflight; - - struct { - SPINLOCK spinlock; - } writer; -} PARSER; - -PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd, PARSER_INPUT_TYPE flags, void *ssl); -void parser_add_keyword(PARSER *working_parser, char *keyword, keyword_function func); -int parser_next(PARSER *working_parser, char *buffer, size_t buffer_size); -int parser_action(PARSER *working_parser, char *input); -void parser_destroy(PARSER *working_parser); - -PARSER_RC pluginsd_set(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_begin(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_end(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_chart(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_flush(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_disable(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_label(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_overwrite(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_clabel_commit(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_clabel(char **words, size_t num_words, void *user); - -PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user); - -PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user); -PARSER_RC pluginsd_end_v2(char **words, size_t num_words, void *user); -void pluginsd_cleanup_v2(void *user); - -#endif |