diff options
| -rw-r--r-- | Makefile.am | 1 | ||||
| -rw-r--r-- | configure.ac | 21 | ||||
| -rw-r--r-- | libnetdata/log/log2journal.c | 1375 | ||||
| -rw-r--r-- | libnetdata/log/log2journal.md | 126 |
4 files changed, 1385 insertions, 138 deletions
diff --git a/Makefile.am b/Makefile.am index f661bffccb..dc253765e6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1323,6 +1323,7 @@ if ENABLE_LOG2JOURNAL log2journal_SOURCES = $(LOG2JOURNAL_FILES) log2journal_LDADD = \ $(OPTIONAL_PCRE2_LIBS) \ + $(OPTIONAL_YAML_LIBS) \ $(NULL) endif diff --git a/configure.ac b/configure.ac index bc3f58bc75..69e6fcd8cd 100644 --- a/configure.ac +++ b/configure.ac @@ -853,6 +853,26 @@ AC_MSG_RESULT([${enable_jsonc}]) AM_CONDITIONAL([ENABLE_JSONC], [test "${enable_jsonc}" = "yes"]) # ----------------------------------------------------------------------------- +# libyaml + +PKG_CHECK_MODULES( + [LIBYAML], + [yaml-0.1], + [AC_CHECK_LIB( + [yaml], + [yaml_parser_initialize], + [have_libyaml=yes], + [have_libyaml=no] + )], + [have_libyaml=no] +) + +if test "x$have_libyaml" = "xyes"; then + AC_DEFINE([HAVE_LIBYAML], [1], [libyaml usability]) + OPTIONAL_YAML_LIBS="-lyaml" +fi + +# ----------------------------------------------------------------------------- # YAML if test -z "${YAML_LIBS}"; then @@ -1983,6 +2003,7 @@ AC_SUBST([OPTIONAL_UV_LIBS]) AC_SUBST([OPTIONAL_LZ4_LIBS]) AC_SUBST([OPTIONAL_BROTLIENC_LIBS]) AC_SUBST([OPTIONAL_BROTLIDEC_LIBS]) +AC_SUBST([OPTIONAL_YAML_LIBS]) AC_SUBST([OPTIONAL_CURL_LIBS]) AC_SUBST([OPTIONAL_PCRE2_LIBS]) AC_SUBST([OPTIONAL_ZSTD_LIBS]) diff --git a/libnetdata/log/log2journal.c b/libnetdata/log/log2journal.c index 68da8b4079..173a7af5bd 100644 --- a/libnetdata/log/log2journal.c +++ b/libnetdata/log/log2journal.c @@ -17,6 +17,10 @@ #define PCRE2_CODE_UNIT_WIDTH 8 #include <pcre2.h> +#ifdef HAVE_LIBYAML +#include <yaml.h> +#endif + #define MAX_OUTPUT_KEYS 1024 #define OVECCOUNT (MAX_OUTPUT_KEYS * 3) // should be a multiple of 3 #define MAX_LINE_LENGTH (1024 * 1024) @@ -28,6 +32,106 @@ #define MAX_KEY_LEN 64 // according to systemd-journald #define MAX_VALUE_LEN (48 * 1024) // according to systemd-journald +struct key_rewrite; +static pcre2_code *jb_compile_pcre2_pattern(const char *pattern); +static bool parse_replacement_pattern(struct key_rewrite *rw); + +#define YAML_CONFIG_NGINX_COMBINED \ + "# Netdata log2journal Configuration Template\n" \ + "# The following parses nginx log files using the combined format.\n" \ + "\n" \ + "# The PCRE2 pattern to match log entries and give names to the fields.\n" \ + "# The journal will have these names, so follow their rules. You can\n" \ + "# initiate an extended PCRE2 pattern by starting the pattern with (?x)\n" \ + "pattern: |\n" \ + " (?x) # Enable PCRE2 extended mode\n" \ + " ^\n" \ + " (?<NGINX_REMOTE_ADDR>[^ ]+) \\s - \\s # NGINX_REMOTE_ADDR\n" \ + " (?<NGINX_REMOTE_USER>[^ ]+) \\s # NGINX_REMOTE_USER\n" \ + " \\[\n" \ + " (?<NGINX_TIME_LOCAL>[^\\]]+) # NGINX_TIME_LOCAL\n" \ + " \\]\n" \ + " \\s+ \"\n" \ + " (?<MESSAGE>\n" \ + " (?<NGINX_METHOD>[A-Z]+) \\s+ # NGINX_METHOD\n" \ + " (?<NGINX_URL>[^ ]+) \\s+\n" \ + " HTTP/(?<NGINX_HTTP_VERSION>[^\"]+)\n" \ + " )\n" \ + " \" \\s+\n" \ + " (?<NGINX_STATUS>\\d+) \\s+ # NGINX_STATUS\n" \ + " (?<NGINX_BODY_BYTES_SENT>\\d+) \\s+ # NGINX_BODY_BYTES_SENT\n" \ + " \"(?<NGINX_HTTP_REFERER>[^\"]*)\" \\s+ # NGINX_HTTP_REFERER\n" \ + " \"(?<NGINX_HTTP_USER_AGENT>[^\"]*)\" # NGINX_HTTP_USER_AGENT\n" \ + "\n" \ + "# When log2journal can detect the filename of each log entry (tail gives it\n" \ + "# only when it tails multiple files), this key will be used to send the\n" \ + "# filename to the journals.\n" \ + "filename:\n" \ + " key: NGINX_LOG_FILENAME\n" \ + "\n" \ + "# Duplicate fields under a different name. You can duplicate multiple fields\n" \ + "# to a new one and then use rewrite rules to change its value.\n" \ + "duplicate:\n" \ + "\n" \ + " # we insert the field PRIORITY as a copy of NGINX_STATUS.\n" \ + " - key: PRIORITY\n" \ + " values_of:\n" \ + " - NGINX_STATUS\n" \ + "\n" \ + " # we inject the field NGINX_STATUS_FAMILY as a copy of NGINX_STATUS.\n" \ + " - key: NGINX_STATUS_FAMILY\n" \ + " values_of: \n" \ + " - NGINX_STATUS\n" \ + "\n" \ + "# Inject constant fields into the journal logs.\n" \ + "inject:\n" \ + " - key: SYSLOG_IDENTIFIER\n" \ + " value: \"nginx-log\"\n" \ + "\n" \ + "# Rewrite the value of fields (including the duplicated ones).\n" \ + "# The search pattern can have named groups, and the replace pattern can use\n" \ + "# them as ${name}.\n" \ + "rewrite:\n" \ + " # PRIORTY is a duplicate of NGINX_STATUS\n" \ + " # Valid PRIORITIES: 0=emerg, 1=alert, 2=crit, 3=error, 4=warn, 5=notice, 6=info, 7=debug\n" \ + " - key: \"PRIORITY\"\n" \ + " search: \"^[123]\"\n" \ + " replace: 6\n" \ + "\n" \ + " - key: \"PRIORITY\"\n" \ + " search: \"^4\"\n" \ + " replace: 5\n" \ + "\n" \ + " - key: \"PRIORITY\"\n" \ + " search: \"^5\"\n" \ + " replace: 3\n" \ + "\n" \ + " - key: \"PRIORITY\"\n" \ + " search: \".*\"\n" \ + " replace: 4\n" \ + " \n" \ + " # NGINX_STATUS_FAMILY is a duplicate of NGINX_STATUS\n" \ + " - key: \"NGINX_STATUS_FAMILY\"\n" \ + " search: \"^(?<first_digit>[1-5])\"\n" \ + " replace: \"${first_digit}xx\"\n" \ + "\n" \ + " - key: \"NGINX_STATUS_FAMILY\"\n" \ + " search: \".*\"\n" \ + " replace: \"UNKNOWN\"\n" \ + "\n" \ + "# Control what to do when input logs do not match the main PCRE2 pattern.\n" \ + "unmatched:\n" \ + " # The journal key to log the PCRE2 error message to.\n" \ + " # Set this to MESSAGE, so you to see the error in the log.\n" \ + " key: MESSAGE\n" \ + " \n" \ + " # Inject static fields to the unmatched entries.\n" \ + " # Set PRIORITY=1 (alert) to help you spot unmatched entries in the logs.\n" \ + " inject:\n" \ + " - key: PRIORITY\n" \ + " value: 1\n" \ + "\n" + void display_help(const char *name) { printf("\n"); printf("Netdata log2journal " PACKAGE_VERSION "\n"); @@ -41,46 +145,57 @@ void display_help(const char *name) { printf("\n"); printf("Options:\n"); printf("\n"); - printf(" --filename-key=KEY\n"); + printf(" --file /path/to/file.yaml\n"); + printf(" Read yaml configuration file for instructions.\n"); + printf("\n"); + printf(" --config CONFIG_NAME\n"); + printf(" Run with the internal configuration named CONFIG_NAME\n"); + printf(" Available internal configs: nginx-combined\n"); + printf("\n"); + printf(" --show-config\n"); + printf(" Show the configuration in yaml format before starting the job.\n"); + printf(" This is also an easy way to convert command line parameters to yaml.\n"); + printf("\n"); + printf(" --filename-key KEY\n"); printf(" Add a field with KEY as the key and the current filename as value.\n"); printf(" Automatically detects filenames when piped after 'tail -F',\n"); printf(" and tail matches multiple filenames.\n"); printf(" To inject the filename when tailing a single file, use --inject.\n"); printf("\n"); - printf(" --unmatched-key=KEY\n"); + printf(" --unmatched-key KEY\n"); printf(" Include unmatched log entries in the output with KEY as the field name.\n"); printf(" Use this to include unmatched entries to the output stream.\n"); printf(" Usually it should be set to --unmatched-key=MESSAGE so that the\n"); printf(" unmatched entry will appear as the log message in the journals.\n"); printf(" Use --inject-unmatched to inject additional fields to unmatched lines.\n"); printf("\n"); - printf(" --duplicate=TARGET=KEY1[,KEY2[,KEY3[,...]]\n"); + printf(" --duplicate TARGET=KEY1[,KEY2[,KEY3[,...]]\n"); printf(" Create a new key called TARGET, duplicating the values of the keys\n"); printf(" given. Useful for further processing. When multiple keys are given,\n"); printf(" their values are separated by comma.\n"); printf(" Up to %d duplications can be given on the command line, and up to\n", MAX_KEY_DUPS); printf(" %d keys per duplication command are allowed.\n", MAX_KEY_DUPS_KEYS); printf("\n"); - printf(" --inject=LINE\n"); + printf(" --inject LINE\n"); printf(" Inject constant fields to the output (both matched and unmatched logs).\n"); printf(" --inject entries are added to unmatched lines too, when their key is\n"); printf(" not used in --inject-unmatched (--inject-unmatched override --inject).\n"); printf(" Up to %d fields can be injected.\n", MAX_INJECTIONS); printf("\n"); - printf(" --inject-unmatched=LINE\n"); + printf(" --inject-unmatched LINE\n"); printf(" Inject lines into the output for each unmatched log entry.\n"); printf(" Usually, --inject-unmatched=PRIORITY=3 is needed to mark the unmatched\n"); printf(" lines as errors, so that they can easily be spotted in the journals.\n"); printf(" Up to %d such lines can be injected.\n", MAX_INJECTIONS); printf("\n"); - printf(" --rewrite=KEY=/SearchPattern/ReplacePattern\n"); + printf(" --rewrite KEY=/SearchPattern/ReplacePattern\n"); printf(" Apply a rewrite rule to the values of a specific key.\n"); printf(" The first character after KEY= is the separator, which should also\n"); printf(" be used between the search pattern and the replacement pattern.\n"); printf(" The search pattern is a PCRE2 regular expression, and the replacement\n"); printf(" pattern supports literals and named capture groups from the search pattern.\n"); printf(" Example:\n"); - printf(" --rewrite=DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n"); + printf(" --rewrite DATE=/^(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})$/\n"); printf(" ${day}/${month}/${year}\n"); printf(" This will rewrite dates in the format YYYY-MM-DD to DD/MM/YYYY.\n"); printf("\n"); @@ -105,6 +220,8 @@ void display_help(const char *name) { printf(" are usually valid PCRE2 patterns too.\n"); printf(" Regular expressions without named groups are ignored.\n"); printf("\n"); + printf("The program accepts all parameters as both --option=value and --option value.\n"); + printf("\n"); printf("The maximum line length accepted is %d characters.\n", MAX_LINE_LENGTH); printf("The maximum number of fields in the PCRE2 pattern is %d.\n", OVECCOUNT / 3); printf("\n"); @@ -166,9 +283,16 @@ void display_help(const char *name) { printf("\n"); printf("You can find the most common fields at 'man systemd.journal-fields'.\n"); printf("\n"); + printf("Example YAML file:\n\n" + "--------------------------------------------------------------------------------\n" + "%s" + "--------------------------------------------------------------------------------\n" + "\n", + YAML_CONFIG_NGINX_COMBINED); } // ---------------------------------------------------------------------------- +// logging // enable the compiler to check for printf like errors on our log2stderr() function static void log2stderr(const char *format, ...) __attribute__ ((format(__printf__, 1, 2))); @@ -181,6 +305,41 @@ static void log2stderr(const char *format, ...) { } // ---------------------------------------------------------------------------- +// allocation functions abstraction + +void *mallocz(size_t size) { + void *ptr = malloc(size); + if (!ptr) { + log2stderr("Fatal Error: Memory allocation failed. Requested size: %zu bytes.", size); + exit(EXIT_FAILURE); + } + return ptr; +} + +char *strdupz(const char *s) { + char *ptr = strdup(s); + if (!ptr) { + log2stderr("Fatal Error: Memory allocation failed in strdup."); + exit(EXIT_FAILURE); + } + return ptr; +} + +char *strndupz(const char *s, size_t n) { + char *ptr = strndup(s, n); + if (!ptr) { + log2stderr("Fatal Error: Memory allocation failed in strndup. Requested size: %zu bytes.", n); + exit(EXIT_FAILURE); + } + return ptr; +} + +void freez(void *ptr) { + if (ptr) + free(ptr); +} + +// ---------------------------------------------------------------------------- size_t copy_to_buffer(char *dst, size_t dst_size, const char *src, size_t src_len) { if(dst_size < 2) { @@ -210,6 +369,11 @@ typedef struct txt { } TEXT; static void txt_replace(TEXT *txt, const char *s, size_t len) { + if(!s || !*s || len == 0) { + s = ""; + len = 0; + } + if(len + 1 <= txt->size) { // the existing value allocation, fits our value @@ -220,9 +384,9 @@ static void txt_replace(TEXT *txt, const char *s, size_t len) { // no existing value allocation, or too small for our value if(txt->s) - free(txt->s); + freez(txt->s); - txt->s = strndup(s, len); + txt->s = strndupz(s, len); txt->size = len + 1; } } @@ -269,6 +433,8 @@ struct key_rewrite { }; struct log_job { + bool show_config; + const char *pattern; struct { @@ -301,29 +467,98 @@ struct log_job { } rewrites; }; +static bool log_job_add_filename_key(struct log_job *jb, const char *key, size_t key_len) { + if(!key || !*key) { + log2stderr("filename key cannot be empty."); + return false; + } + + if(jb->filename.key) + freez((char*)jb->filename.key); + + jb->filename.key = strndupz(key, key_len); + + return true; +} + +static bool log_job_add_injection(struct log_job *jb, const char *key, size_t key_len, const char *value, size_t value_len, bool unmatched) { + if (unmatched) { + if (jb->unmatched.injections.used >= MAX_INJECTIONS) { + log2stderr("Error: too many unmatched injections. You can inject up to %d lines.", MAX_INJECTIONS); + return false; + } + } + else { + if (jb->injections.used >= MAX_INJECTIONS) { + log2stderr("Error: too many injections. You can inject up to %d lines.", MAX_INJECTIONS); + return false; + } + } + + if (unmatched) { + key_value_replace(&jb->unmatched.injections.keys[jb->unmatched.injections.used++], + key, key_len, + value, value_len); + } else { + key_value_replace(&jb->injections.keys[jb->injections.used++], + key, key_len, + value, value_len); + } + + return true; +} + +static bool log_job_add_rewrite(struct log_job *jb, const char *key, const char *search_pattern, const char *replace_pattern) { + pcre2_code *re = jb_compile_pcre2_pattern(search_pattern); + if (!re) { + return false; + } + + struct key_rewrite *rw = &jb->rewrites.array[jb->rewrites.used++]; + rw->key = strdupz(key); + rw->hash = XXH3_64bits(rw->key, strlen(rw->key)); + rw->search_pattern = strdupz(search_pattern); + rw->replace_pattern = strdupz(replace_pattern); + rw->re = re; + rw->match_data = pcre2_match_data_create_from_pattern(rw->re, NULL); + + // Parse the replacement pattern and create the linked list + if (!parse_replacement_pattern(rw)) { + pcre2_match_data_free(rw->match_data); + pcre2_code_free(rw->re); + freez(rw->key); + freez(rw->search_pattern); + freez(rw->replace_pattern); + jb->rewrites.used--; + return false; + } + + return true; +} + void jb_cleanup(struct log_job *jb) { for(size_t i = 0; i < jb->injections.used ;i++) { if(jb->injections.keys[i].value.s) - free(jb->injections.keys[i].value.s); + freez(jb->injections.keys[i].value.s); } for(size_t i = 0; i < jb->unmatched.injections.used ;i++) { if(jb->unmatched.injections.keys[i].value.s) - free(jb->unmatched.injections.keys[i].value.s); + freez(jb->unmatched.injections.keys[i].value.s); } for(size_t i = 0; i < jb->dups.used ;i++) { struct key_dup *kd = &jb->dups.array[i]; if(kd->target) - free(kd->target); + freez(kd->target); for(size_t j = 0; j < kd->used ; j++) { if (kd->keys[j]) - free(kd->keys[j]); + freez(kd->keys[j]); if (kd->values[j].s) - free(kd->values[j].s); + freez(kd->values[j].s); } } @@ -331,13 +566,13 @@ void jb_cleanup(struct log_job *jb) { struct key_rewrite *rw = &jb->rewrites.array[i]; if (rw->key) - free(rw->key); + freez(rw->key); if (rw->search_pattern) - free(rw->search_pattern); + freez(rw->search_pattern); if (rw->replace_pattern) - free(rw->replace_pattern); + freez(rw->replace_pattern); if(rw->match_data) pcre2_match_data_free(rw->match_data); @@ -351,9 +586,9 @@ void jb_cleanup(struct log_job *jb) { struct replacement_node *next = current->next; if (current->s) - free((void *)current->s); + freez((void *)current->s); - free(current); + freez(current); current = next; } } @@ -466,10 +701,747 @@ static inline void send_key_value_constant(struct log_job *jb, const char *key, } // ---------------------------------------------------------------------------- + +static struct key_dup *add_duplicate_target_to_job(struct log_job *jb, const char *target, size_t target_len) { + if (jb->dups.used >= MAX_KEY_DUPS) { + log2stderr("Error: Too many duplicates defined. Maximum allowed is %d.", MAX_KEY_DUPS); + return NULL; + } + + struct key_dup *kd = &jb->dups.array[jb->dups.used++]; + kd->target = strndupz(target, target_len); + kd->hash = XXH3_64bits(kd->target, target_len); + kd->used = 0; + kd->exposed = false; + + // Initialize values array + for (size_t i = 0; i < MAX_KEY_DUPS_KEYS; i++) { + kd->values[i].s = NULL; + kd->values[i].size = 0; + } + + return kd; +} + +static bool add_key_to_duplicate(struct key_dup *kd, const char *key, size_t key_len) { + if (kd->used >= MAX_KEY_DUPS_KEYS) { + log2stderr("Error: Too many keys in duplication of target '%s'.", kd->target); + return false; + } + + kd->keys[kd->used++] = strndupz(key, key_len); + return true; +} + +// ---------------------------------------------------------------------------- +// yaml configuration file + +#ifdef HAVE_LIBYAML + + +// ---------------------------------------------------------------------------- +// yaml library functions + +static const char *yaml_event_name(yaml_event_type_t type) { + switch (type) { + case YAML_NO_EVENT: + return "YAML_NO_EVENT"; + + case YAML_SCALAR_EVENT: + return "YAML_SCALAR_EVENT"; + + case YAML_ALIAS_EVENT: + return "YAML_ALIAS_EVENT"; + + case YAML_MAPPING_START_EVENT: + return "YAML_MAPPING_START_EVENT"; + + case YAML_MAPPING_END_EVENT: + return "YAML_MAPPING_END_EVENT"; + + case YAML_SEQUENCE_START_EVENT: + return "YAML_SEQUENCE_START_EVENT"; + + case YAML_SEQUENCE_END_EVENT: + return "YAML_SEQUENCE_END_EVENT"; + + case YAML_STREAM_START_EVENT: + return "YAML_STREAM_START_EVENT"; + + case YAML_STREAM_END_EVENT: + return "YAML_STREAM_END_EVENT"; + + case YAML_DOCUMENT_START_EVENT: + return "YAML_DOCUMENT_START_EVENT"; + + case YAML_DOCUMENT_END_EVENT: + return "YAML_DOCUMENT_END_EVENT"; + + default: + return "UNKNOWN"; + } +} + +#define yaml_error(parser, event, fmt, args...) yaml_error_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__, fmt, ##args) +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) __attribute__ ((format(__printf__, 6, 7))); +static void yaml_error_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file, const char *format, ...) { + char buf[1024] = ""; // Initialize buf to an empty string + const char *type = ""; + + if(event) { + type = yaml_event_name(event->type); + + switch (event->type) { + case YAML_SCALAR_EVENT: + copy_to_buffer(buf, sizeof(buf), (char *)event->data.scalar.value, event->data.scalar.length); + break; + + case YAML_ALIAS_EVENT: + snprintf(buf, sizeof(buf), "%s", event->data.alias.anchor); + break; + + default: + break; + } + } + + fprintf(stderr, "YAML %zu@%s, %s(): (line %d, column %d, %s%s%s): ", + line, file, function, + (int)(parser->mark.line + 1), (int)(parser->mark.column + 1), + type, buf[0]? ", near ": "", buf); + + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + fprintf(stderr, "\n"); +} + +#define yaml_parse(parser, event) yaml_parse_with_trace(parser, event, __LINE__, __FUNCTION__, __FILE__) +static bool yaml_parse_with_trace(yaml_parser_t *parser, yaml_event_t *event, size_t line, const char *function, const char *file) { + if (!yaml_parser_parse(parser, event)) { + yaml_error(parser, NULL, "YAML parser error %d", parser->error); + return false; + } + +// fprintf(stderr, ">>> %s >>> %.*s\n", +// yaml_event_name(event->type), +// event->type == YAML_SCALAR_EVENT ? event->data.scalar.length : 0, +// event->type == YAML_SCALAR_EVENT ? (char *)event->data.scalar.value : ""); + + return true; +} + +#define yaml_parse_expect_event(parser, type) yaml_parse_expect_event_with_trace(parser, type, __LINE__, __FUNCTION__, __FILE__) +static bool yaml_parse_expect_event_with_trace(yaml_parser_t *parser, yaml_event_type_t type, size_t line, const char *function, const char *file) { + yaml_event_t event; + if (!yaml_parse(parser, &event)) + return false; + + bool ret = true; + if(event.type != type) { + yaml_error_with_trace(parser, &event, line, function, file, "unexpected event - expecting: %s", yaml_event_name(type)); + ret = false; + } +// else +// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function); + + yaml_event_delete(&event); + return ret; +} + +#define yaml_scalar_matches(event, s, len) yaml_scalar_matches_with_trace(event, s, len, __LINE__, __FUNCTION__, __FILE__) +static bool yaml_scalar_matches_with_trace(yaml_event_t *event, const char *s, size_t len, size_t line __maybe_unused, const char *function __maybe_unused, const char *file __maybe_unused) { + if(event->type != YAML_SCALAR_EVENT) + return false; + + if(len != event->data.scalar.length) + return false; +// else +// fprintf(stderr, "OK (%zu@%s, %s()\n", line, file, function); + + return strcmp((char *)event->data.scalar.value, s) == 0; +} + +// ---------------------------------------------------------------------------- + +static struct key_dup *yaml_parse_duplicate_key(struct log_job *jb, yaml_parser_t *parser) { + yaml_event_t event; + + if (!yaml_parse(parser, &event)) + return false; + + struct key_dup *kd = NULL; + if(event.type == YAML_SCALAR_EVENT) { + kd = add_duplicate_target_to_job(jb, (char *)event.data.scalar.value, event.data.scalar.length); + } + else + yaml_error(parser, &event, "duplicate key must be a scalar."); + + yaml_event_delete(&event); + return kd; +} + +static size_t yaml_parse_duplicate_from(struct log_job *jb, yaml_parser_t *parser, struct key_dup *kd) { + size_t errors = 0; + yaml_event_t event; + + if (!yaml_parse(parser, &event)) + return 1; + + bool ret = true; + if(event.type == YAML_SCALAR_EVENT) + ret = add_key_to_duplicate(kd, (char *)event.data.scalar.value, event.data.scalar.length); + + else if(event.type == YAML_SEQUENCE_START_EVENT) { + bool finished = false; + while(!errors && !finished) { + yaml_event_t sub_event; + if (!yaml_parse(parser, &sub_event)) + return errors++; + else { + if (sub_event.type == YAML_SCALAR_EVENT) + add_key_to_duplicate(kd, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length); + + else if (sub_event.type == YAML_SEQUENCE_END_EVENT) + finished = true; + + yaml_event_delete(&sub_event); + } + } + } + else + yaml_error(parser, &event, "not expected event type"); + + yaml_event_delete(&event); + return errors; +} + +static size_t yaml_parse_filename_injection(yaml_parser_t *parser, struct log_job *jb) { + yaml_event_t event; + size_t errors = 0; + + if(!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT)) + return 1; + + if (!yaml_parse(parser, &event)) + return 1; + + if (yaml_scalar_matches(&event, "key", strlen("key"))) { + yaml_event_t sub_event; + if (!yaml_parse(parser, &sub_event)) + errors++; + + else { + if (event.type == YAML_SCALAR_EVENT) { + if(!log_job_add_filename_key(jb, (char *)sub_event.data.scalar.value, sub_event.data.scalar.length)) + errors++; + } + + else { + yaml_error(parser, &sub_event, "expected the filename as %s", yaml_event_name(YAML_SCALAR_EVENT)); + errors++; + } + + yaml_event_delete(&sub_event); + } + } + + if(!yaml_parse_expect_event(parser, YAML_MAPPING_END_EVENT)) + errors++; + + yaml_event_delete(&event); + return errors; +} + +static size_t yaml_parse_duplicates_injection(yaml_parser_t *parser, struct log_job *jb) { + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT)) + return 1; + + struct key_dup *kd = NULL; + + // Expecting a key-value pair for each duplicate + bool finished; + size_t errors = 0; + while (!errors && !finished) { + yaml_event_t event; + if (!yaml_parse(parser, &event)) { + errors++; + break; + } + + if(event.type == YAML_MAPPING_START_EVENT) { + ; + } + if (event.type == YAML_SEQUENCE_END_EVENT) { + finished = true; + } + else if(event.type == YAML_SCALAR_EVENT) { + if (yaml_scalar_matches(&event, "key", strlen("key"))) { + kd = yaml_parse_duplicate_key(jb, parser); + if (!kd) + errors++; + else { + while (!errors && kd) { + yaml_event_t sub_event; + if (!yaml_parse(parser, &sub_event)) { + errors++; + break; + } + + if (sub_event.type == YAML_MAPPING_END_EVENT) { + kd = NULL; + } else if (sub_event.type == YAML_SCALAR_EVENT) { + if (yaml_scalar_matches(&sub_event, "values_of", strlen("values_of"))) { + if (!kd) { + yaml_error(parser, &sub_event, "Found 'values_of' but the 'key' is not set."); + errors++; + } else + errors += yaml_parse_duplicate_from(jb, parser, kd); + } else { + yaml_error(parser, &sub_event, "unknown scalar"); + errors++; + } + } else { + yaml_error(parser, &sub_event, "unexpected event type"); + errors++; + } + + // Delete the event after processing + yaml_event_delete(&event); + } + } + } else { + yaml_error(parser, &event, "unknown scalar"); + errors++; + } + } + + yaml_event_delete(&event); + } + + return errors; +} + +static bool yaml_parse_constant_field_injection(yaml_parser_t *parser, struct log_job *jb, bool unmatched) { + yaml_event_t event; + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) { + yaml_error(parser, &event, "Expected scalar for constant field injection key"); + yaml_event_delete(&event); + return false; + } + + char *key = strndupz((char *)event.data.scalar.value, event.data.scalar.length); + char *value = NULL; + bool ret = false; + + yaml_event_delete(&event); + + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) { + yaml_error(parser, &event, "Expected scalar for constant field injection value"); + goto cleanup; + } + + if(!yaml_scalar_matches(&event, "value", strlen("value"))) { + yaml_error(parser, &event, "Expected scalar 'value'"); + goto cleanup; + } + + if (!yaml_parse(parser, &event) || event.type != YAML_SCALAR_EVENT) { + yaml_error(parser, &event, "Expected scalar for constant field injection value"); + goto cleanup; + } + + value = strndupz((char *)event.data.scalar.value, event.data.scalar.length); + + if(!log_job_add_injection(jb, key, strlen(key), value, strlen(value), unmatched)) + ret = false; + else + ret = true; + + ret = true; + +cleanup: + yaml_event_delete(&event); + freez(key); + freez(value); + return !ret ? 1 : 0; +} + +static bool yaml_parse_injection_mapping(yaml_parser_t *parser, struct log_job *jb, bool unmatched) { + yaml_event_t event; + size_t errors = 0; + bool finished = false; + + while (!errors && !finished) { + if (!yaml_parse(parser, &event)) { + errors++; + continue; + } + + switch (event.type) { + case YAML_SCALAR_EVENT: + if (yaml_scalar_matches(&event, "key", strlen("key"))) { + errors += yaml_parse_constant_field_injection(parser, jb, unmatched); + } else { + yaml_error(parser, &event, "Unexpected scalar in injection mapping"); + errors++; + } + break; + + case YAML_MAPPING_END_EVENT: + finished = true; + break; + + default: + yaml_error(parser, &event, "Unexpected event in injection mapping"); + errors++; + break; + } + + yaml_event_delete(&event); + } + + return errors == 0; +} + +static size_t yaml_parse_injections(yaml_parser_t *parser, struct log_job *jb, bool unmatched) { + yaml_event_t event; + size_t errors = 0; + bool finished = false; + + if (!yaml_parse_expect_event(parser, YAML_SEQUENCE_START_EVENT)) + return 1; + + while (!errors && !finished) { + if (!yaml_parse(parser, &event)) { + errors++; + continue; + } + + switch (event.type) { + case YAML_MAPPING_START_EVENT: + if (!yaml_parse_injection_mapping(parser, jb, unmatched)) + errors++; + break; + + case YAML_SEQUENCE_END_EVENT: + finished = true; + break; + + default: + yaml_error(parser, &event, "Unexpected event in injections sequence"); + errors++; + break; + } + + yaml_event_delete(&event); + } + + return errors; +} + +static size_t yaml_parse_unmatched(yaml_parser_t *parser, struct log_job *jb) { + size_t errors = 0; + bool finished = false; + + if (!yaml_parse_expect_event(parser, YAML_MAPPING_START_EVENT)) + return 1; + + while (!errors && !finished) { + yaml_event_t event; + if (!yaml_parse(parser, &event)) { + errors++; + continue; + } + + switch (event.type) { + case YAML_SCALAR_EVENT: + if (yaml_scalar_matches(&event, "key", strlen("key"))) { + yaml_event_t sub_event; + if (!yaml_parse(parser, &sub_event)) { + errors++; + } else { + if (sub_event.type == YAML_SCALAR_EVENT) { + jb->unmatched.key = strndupz((char *)sub_event.data.scalar.value, sub_event.data.scalar.length); + } else { + yaml_error(parser, &sub_event, "expected a scalar value for 'key'"); + errors++; + } + yaml_event_delete(&sub_event); + } + } else if (yaml_scalar_matches(&event, "inject", strlen("inject"))) { + errors += yaml_parse_injections(parser, jb, true); + } else { + yaml_error(parser, &event, "Unexpected scalar in unmatched section"); + errors++; + } + break; + + case YAML_MAPPING_END_EVENT: + finished = true; + break; + + default: + yaml_error(parser, &event, "Unexpected event in unmatched section"); + errors++; + break; + } + + yaml_event_delete(&event); + } + + return errors; +} + |