diff options
99 files changed, 14777 insertions, 58 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6fe613358c..7f368ceb7e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -32,6 +32,7 @@ system/ @Ferroin @tkatsoulas tests/ @Ferroin @vkalintiris @tkatsoulas web/ @thiagoftsm @vkalintiris web/gui/ @novykh +logsmanagement/ @Dim-P @thiagoftsm # Ownership by filetype (overwrites ownership by directory) *.am @Ferroin @tkatsoulas diff --git a/.github/labeler.yml b/.github/labeler.yml index 1dd4d472a1..0ea825ef4b 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -153,3 +153,6 @@ area/tests: area/web: - web/** + +area/logs-management: + - logsmanagement/** diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index 732a4a5a1a..8cb61f2cf8 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -234,6 +234,7 @@ jobs: ./.git/* packaging/makeself/makeself.sh packaging/makeself/makeself-header.sh + ./fluent-bit/* yamllint: name: yamllint diff --git a/.gitignore b/.gitignore index a8e8133388..7daa25e623 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,9 @@ debugfs.plugin systemd-journal.plugin !systemd-journal.plugin/ +logs-management.plugin +!logs-management.plugin/ + # protoc generated files *.pb.cc *.pb.h @@ -153,6 +156,8 @@ collectors/ioping.plugin/ioping.plugin collectors/go.d.plugin web/netdata-switch-dashboard.sh +logsmanagement/stress_test/stress_test + # installer generated files /netdata-uninstaller.sh /netdata-updater.sh diff --git a/.gitmodules b/.gitmodules index 2dae4a1dd2..ac8e131f5d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,8 @@ path = web/server/h2o/libh2o url = https://github.com/h2o/h2o.git ignore = untracked +[submodule "fluent-bit"] + path = fluent-bit + url = https://github.com/fluent/fluent-bit.git + shallow = true + ignore = dirty
\ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 33c9905113..6d6752ea7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1074,6 +1074,44 @@ ELSE() message(STATUS "ML: disabled") ENDIF() +set(LOGSMANAGEMENT_FILES + logsmanagement/rrd_api/rrd_api_docker_ev.c + logsmanagement/rrd_api/rrd_api_docker_ev.h + logsmanagement/rrd_api/rrd_api_generic.c + logsmanagement/rrd_api/rrd_api_generic.h + logsmanagement/rrd_api/rrd_api_kernel.c + logsmanagement/rrd_api/rrd_api_kernel.h + logsmanagement/rrd_api/rrd_api_mqtt.c + logsmanagement/rrd_api/rrd_api_mqtt.h + logsmanagement/rrd_api/rrd_api_stats.c + logsmanagement/rrd_api/rrd_api_stats.h + logsmanagement/rrd_api/rrd_api_systemd.c + logsmanagement/rrd_api/rrd_api_systemd.h + logsmanagement/rrd_api/rrd_api_web_log.c + logsmanagement/rrd_api/rrd_api_web_log.h + logsmanagement/rrd_api/rrd_api.h + logsmanagement/unit_test/unit_test.c + logsmanagement/unit_test/unit_test.h + logsmanagement/circular_buffer.c + logsmanagement/circular_buffer.h + logsmanagement/db_api.c + logsmanagement/db_api.h + logsmanagement/file_info.h + logsmanagement/flb_plugin.c + logsmanagement/flb_plugin.h + logsmanagement/functions.c + logsmanagement/functions.h + logsmanagement/helper.h + logsmanagement/defaults.h + logsmanagement/logsmanag_config.c + logsmanagement/logsmanag_config.h + logsmanagement/logsmanagement.c + logsmanagement/parser.c + logsmanagement/parser.h + logsmanagement/query.c + logsmanagement/query.h + ) + set(NETDATA_FILES collectors/all.h ${DAEMON_FILES} @@ -1113,6 +1151,13 @@ add_definitions( ) # ----------------------------------------------------------------------------- +# logs management + +IF(ENABLE_LOGSMANAGEMENT) + list(APPEND NETDATA_FILES ${LOGSMANAGEMENT_FILES}) +ENDIF() + +# ----------------------------------------------------------------------------- # kinesis exporting connector IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND @@ -1748,7 +1793,6 @@ endif() endif() endif() - # generate config.h so that CMake becomes independent of automake ## netdata version diff --git a/Makefile.am b/Makefile.am index dc253765e6..7638fcfd46 100644 --- a/Makefile.am +++ b/Makefile.am @@ -114,6 +114,7 @@ SUBDIRS += \ web \ claim \ spawn \ + logsmanagement \ $(NULL) AM_CFLAGS = \ @@ -349,6 +350,51 @@ LOG2JOURNAL_FILES = \ libnetdata/log/log2journal.c \ $(NULL) + +LOGSMANAGEMENT_FILES = \ + logsmanagement/circular_buffer.c \ + logsmanagement/circular_buffer.h \ + logsmanagement/db_api.c \ + logsmanagement/db_api.h \ + logsmanagement/defaults.h \ + logsmanagement/file_info.h \ + logsmanagement/flb_plugin.c \ + logsmanagement/flb_plugin.h \ + logsmanagement/functions.c \ + logsmanagement/functions.h \ + logsmanagement/helper.h \ + logsmanagement/logsmanag_config.c \ + logsmanagement/logsmanag_config.h \ + logsmanagement/logsmanagement.c \ + logsmanagement/parser.c \ + logsmanagement/parser.h \ + logsmanagement/query.c \ + logsmanagement/query.h \ + logsmanagement/rrd_api/rrd_api_docker_ev.c \ + logsmanagement/rrd_api/rrd_api_docker_ev.h \ + logsmanagement/rrd_api/rrd_api_generic.c \ + logsmanagement/rrd_api/rrd_api_generic.h \ + logsmanagement/rrd_api/rrd_api_kernel.c \ + logsmanagement/rrd_api/rrd_api_kernel.h \ + logsmanagement/rrd_api/rrd_api_mqtt.c \ + logsmanagement/rrd_api/rrd_api_mqtt.h \ + logsmanagement/rrd_api/rrd_api_stats.c \ + logsmanagement/rrd_api/rrd_api_stats.h \ + logsmanagement/rrd_api/rrd_api_systemd.c \ + logsmanagement/rrd_api/rrd_api_systemd.h \ + logsmanagement/rrd_api/rrd_api_web_log.c \ + logsmanagement/rrd_api/rrd_api_web_log.h \ + logsmanagement/rrd_api/rrd_api.h \ + database/sqlite/sqlite3.c \ + database/sqlite/sqlite3.h \ + $(LIBNETDATA_FILES) \ + $(NULL) + +LOGSMANAGEMENT_TESTS_FILES = \ + logsmanagement/unit_test/unit_test.c \ + logsmanagement/unit_test/unit_test.h \ + $(NULL) + CUPS_PLUGIN_FILES = \ collectors/cups.plugin/cups_plugin.c \ $(LIBNETDATA_FILES) \ @@ -1342,6 +1388,17 @@ systemd_cat_native_LDADD = \ $(NETDATA_COMMON_LIBS) \ $(NULL) +if ENABLE_LOGSMANAGEMENT + plugins_PROGRAMS += logs-management.plugin + logs_management_plugin_SOURCES = $(LOGSMANAGEMENT_FILES) +if ENABLE_LOGSMANAGEMENT_TESTS + logs_management_plugin_SOURCES += $(LOGSMANAGEMENT_TESTS_FILES) +endif + logs_management_plugin_LDADD = \ + $(NETDATA_COMMON_LIBS) \ + $(NULL) +endif + if ENABLE_PLUGIN_EBPF plugins_PROGRAMS += ebpf.plugin ebpf_plugin_SOURCES = $(EBPF_PLUGIN_FILES) diff --git a/build_external/clean-install-arch-debug.Dockerfile b/build_external/clean-install-arch-debug.Dockerfile index 5a67bfbc39..a3080ea693 100644 --- a/build_external/clean-install-arch-debug.Dockerfile +++ b/build_external/clean-install-arch-debug.Dockerfile @@ -48,9 +48,10 @@ RUN find . -type f >/opt/netdata/manifest RUN CFLAGS="-Og -g -ggdb -Wall -Wextra -Wformat-signedness -DNETDATA_INTERNAL_CHECKS=1\ -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto -RUN ln -sf /dev/stdout /var/log/netdata/access.log -RUN ln -sf /dev/stdout /var/log/netdata/debug.log -RUN ln -sf /dev/stderr /var/log/netdata/error.log +RUN ln -sf /dev/stdout /var/log/netdata/access.log && \ + ln -sf /dev/stdout /var/log/netdata/debug.log && \ + ln -sf /dev/stderr /var/log/netdata/error.log && \ + ln -sf /dev/stdout /var/log/netdata/fluentbit.log RUN printf >/opt/netdata/source/gdb_batch '\ set args -D \n\ diff --git a/build_external/clean-install-arch-extras.Dockerfile b/build_external/clean-install-arch-extras.Dockerfile index 8c6f4fbaa2..8b18057e99 100644 --- a/build_external/clean-install-arch-extras.Dockerfile +++ b/build_external/clean-install-arch-extras.Dockerfile @@ -48,9 +48,10 @@ RUN find . -type f >/opt/netdata/manifest RUN CFLAGS="-Og -g -ggdb -Wall -Wextra -Wformat-signedness -DNETDATA_INTERNAL_CHECKS=1\ -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto -RUN ln -sf /dev/stdout /var/log/netdata/access.log -RUN ln -sf /dev/stdout /var/log/netdata/debug.log -RUN ln -sf /dev/stderr /var/log/netdata/error.log +RUN ln -sf /dev/stdout /var/log/netdata/access.log && \ + ln -sf /dev/stdout /var/log/netdata/debug.log && \ + ln -sf /dev/stderr /var/log/netdata/error.log && \ + ln -sf /dev/stdout /var/log/netdata/fluentbit.log RUN rm /var/lib/netdata/registry/netdata.public.unique.id diff --git a/build_external/clean-install-arch.Dockerfile b/build_external/clean-install-arch.D |