From c3dfbe52a61dd0d1995bc420b0e0576cf058fd74 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Tue, 28 Jun 2022 17:04:37 +0300 Subject: netdata doubles (#13217) * netdata doubles * fix cmocka test * fix cmocka test again * fix left-overs of long double to NETDATA_DOUBLE * RRDDIM detached from disk representation; db settings in [db] section of netdata.conf * update the memory before saving * rrdset is now detached from file structures too * on memory mode map, update the memory mapped structures on every iteration * allow RRD_ID_LENGTH_MAX to be changed * granularity secs, back to update every * fix formatting * more formatting --- daemon/config/README.md | 78 +++++++------- daemon/global_statistics.c | 2 +- daemon/main.c | 135 ++++++++++++++++++------ daemon/unit_test.c | 251 +++++++++++++++++++++++---------------------- 4 files changed, 279 insertions(+), 187 deletions(-) (limited to 'daemon') diff --git a/daemon/config/README.md b/daemon/config/README.md index 72f688543a..a44d31e28c 100644 --- a/daemon/config/README.md +++ b/daemon/config/README.md @@ -26,20 +26,21 @@ the [web server access lists](/web/server/README.md#access-lists). `netdata.conf` has sections stated with `[section]`. You will see the following sections: 1. `[global]` to [configure](#global-section-options) the [Netdata daemon](/daemon/README.md). -2. `[directories]` to [configure](#directories-section-options) the directories used by Netdata. -3. `[logs]` to [configure](#logs-section-options) the Netdata logging. -4. `[environment variables]` to [configure](#environment-variables-section-options) the environment variables used +2. `[db]` to [configure](#db-section-options) the database of Netdata. +3. `[directories]` to [configure](#directories-section-options) the directories used by Netdata. +4. `[logs]` to [configure](#logs-section-options) the Netdata logging. +5. `[environment variables]` to [configure](#environment-variables-section-options) the environment variables used Netdata. -5. `[sqlite]` to [configure](#sqlite-section-options) the [Netdata daemon](/daemon/README.md) SQLite settings. -6. `[ml]` to configure settings for [machine learning](/ml/README.md). -7. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md). -8. `[web]` to [configure the web server](/web/server/README.md). -9. `[registry]` for the [Netdata registry](/registry/README.md). -10. `[global statistics]` for the [Netdata registry](/registry/README.md). -11. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md). -12. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH +6. `[sqlite]` to [configure](#sqlite-section-options) the [Netdata daemon](/daemon/README.md) SQLite settings. +7. `[ml]` to configure settings for [machine learning](/ml/README.md). +8. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md). +9. `[web]` to [configure the web server](/web/server/README.md). +10. `[registry]` for the [Netdata registry](/registry/README.md). +11. `[global statistics]` for the [Netdata registry](/registry/README.md). +12. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md). +13. `[plugins]` to [configure](#plugins-section-options) which [collectors](/collectors/README.md) to use and PATH settings. -13. `[plugin:NAME]` sections for each collector plugin, under the +14. `[plugin:NAME]` sections for each collector plugin, under the comment [Per plugin configuration](#per-plugin-configuration). The configuration file is a `name = value` dictionary. Netdata will not complain if you set options unknown to it. When @@ -67,30 +68,35 @@ Please note that your data history will be lost if you have modified `history` p ### [global] section options -| setting | default | info | -|:-------------------------------------:|:------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| process scheduling policy | `keep` | See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy) | -| OOM score | `0` | | -| glibc malloc arena max for plugins | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). | -| glibc malloc arena max for Netdata | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). | -| hostname | auto-detected | The hostname of the computer running Netdata. | -| history | `3996` | Used with `memory mode = save/map/ram/alloc`, not the default `memory mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. This setting can also be configured per chart. Check [Memory Requirements](/database/README.md) for more information. | -| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). | -| memory mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size` and `dbengine disk space`.
`save`: Netdata will save its round robin database on exit and load it on startup.
`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`).
`ram`: The round-robin database will be temporary and it will be lost when Netdata exits.
`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. | -| page cache size | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. | -| dbengine disk space | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. | -| dbengine multihost disk space | 256 | Same functionality as `dbengine disk space`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. | -| host access prefix | | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). | -| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) | -| timezone | auto-detected | The timezone retrieved from the environment variable | -| run as user | `netdata` | The user Netdata will run as. | -| pthread stack size | auto-detected | | -| cleanup obsolete charts after seconds | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions | -| gap when lost iterations above | `1` | | -| cleanup orphan hosts after seconds | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. | -| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions | -| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. | -| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. | +| setting | default | info | +|:-------------------------------------:|:-------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| process scheduling policy | `keep` | See [Netdata process scheduling policy](/daemon/README.md#netdata-process-scheduling-policy) | +| OOM score | `0` | | +| glibc malloc arena max for plugins | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). | +| glibc malloc arena max for Netdata | `1` | See [Virtual memory](/daemon/README.md#virtual-memory). | +| hostname | auto-detected | The hostname of the computer running Netdata. | +| host access prefix | empty | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43). | +| timezone | auto-detected | The timezone retrieved from the environment variable | +| run as user | `netdata` | The user Netdata will run as. | +| pthread stack size | auto-detected | | + +### [db] section options + +| setting | default | info | +|:----------------------------------:|:----------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mode | `dbengine` | `dbengine`: The default for long-term metrics storage with efficient RAM and disk usage. Can be extended with `page cache size MB` and `dbengine disk space MB`.
`save`: Netdata will save its round robin database on exit and load it on startup.
`map`: Cache files will be updated in real-time. Not ideal for systems with high load or slow disks (check `man mmap`).
`ram`: The round-robin database will be temporary and it will be lost when Netdata exits.
`none`: Disables the database at this host, and disables health monitoring entirely, as that requires a database of metrics. | +| retention | `3600` | Used with `mode = save/map/ram/alloc`, not the default `mode = dbengine`. This number reflects the number of entries the `netdata` daemon will by default keep in memory for each chart dimension. Check [Memory Requirements](/database/README.md) for more information. | +| update every | `1` | The frequency in seconds, for data collection. For more information see the [performance guide](/docs/guides/configure/performance.md). | +| page cache size MB | 32 | Determines the amount of RAM in MiB that is dedicated to caching Netdata metric values. | +| dbengine disk space MB | 256 | Determines the amount of disk space in MiB that is dedicated to storing Netdata metric values and all related metadata describing them. | +| dbengine multihost disk space MB | 256 | Same functionality as `dbengine disk space MB`, but includes support for storing metrics streamed to a parent node by its children. Can be used in single-node environments as well. | +| memory deduplication (ksm) | `yes` | When set to `yes`, Netdata will offer its in-memory round robin database and the dbengine page cache to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](/database/README.md#ksm) | +| cleanup obsolete charts after secs | `3600` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also sets the timeout for cleaning up obsolete dimensions | +| gap when lost iterations above | `1` | | +| cleanup orphan hosts after secs | `3600` | How long to wait until automatically removing from the DB a remote Netdata host (child) that is no longer sending data. | +| delete obsolete charts files | `yes` | See [monitoring ephemeral containers](/collectors/cgroups.plugin/README.md#monitoring-ephemeral-containers), also affects the deletion of files for obsolete dimensions | +| delete orphan hosts files | `yes` | Set to `no` to disable non-responsive host removal. | +| enable zero metrics | `no` | Set to `yes` to show charts when all their metrics are zero. | ### [directories] section options diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index b5740b176f..1d268ce81f 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -1146,7 +1146,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) { if(wu->workers_cpu_registered == 0) rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0); else - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (calculated_number)wu->workers_cpu_registered )); + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (NETDATA_DOUBLE)wu->workers_cpu_registered )); rrdset_done(wu->st_workers_cpu); } diff --git a/daemon/main.c b/daemon/main.c index 48d22f00c0..f2714e08eb 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -522,6 +522,58 @@ static void backwards_compatible_config() { config_move(CONFIG_SECTION_STATSD, "enabled", CONFIG_SECTION_PLUGINS, "statsd"); + + config_move(CONFIG_SECTION_GLOBAL, "memory mode", + CONFIG_SECTION_DB, "mode"); + + config_move(CONFIG_SECTION_GLOBAL, "history", + CONFIG_SECTION_DB, "retention"); + + config_move(CONFIG_SECTION_GLOBAL, "update every", + CONFIG_SECTION_DB, "update every"); + + config_move(CONFIG_SECTION_GLOBAL, "page cache size", + CONFIG_SECTION_DB, "page cache size MB"); + + config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc", + CONFIG_SECTION_DB, "page cache with malloc"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine disk space", + CONFIG_SECTION_DB, "dbengine disk space MB"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", + CONFIG_SECTION_DB, "dbengine multihost disk space MB"); + + config_move(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", + CONFIG_SECTION_DB, "memory deduplication (ksm)"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout", + CONFIG_SECTION_DB, "dbengine page fetch timeout secs"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries", + CONFIG_SECTION_DB, "dbengine page fetch retries"); + + config_move(CONFIG_SECTION_GLOBAL, "dbengine extent pages", + CONFIG_SECTION_DB, "dbengine pages per extent"); + + config_move(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", + CONFIG_SECTION_DB, "cleanup obsolete charts after secs"); + + config_move(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", + CONFIG_SECTION_DB, "gap when lost iterations above"); + + config_move(CONFIG_SECTION_GLOBAL, "cleanup orphan hosts after seconds", + CONFIG_SECTION_DB, "cleanup orphan hosts after secs"); + + config_move(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", + CONFIG_SECTION_DB, "delete obsolete charts files"); + + config_move(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", + CONFIG_SECTION_DB, "delete orphan hosts files"); + + config_move(CONFIG_SECTION_GLOBAL, "enable zero metrics", + CONFIG_SECTION_DB, "enable zero metrics"); + } static void get_netdata_configured_variables() { @@ -539,28 +591,40 @@ static void get_netdata_configured_variables() { debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname); // ------------------------------------------------------------------------ - // get default database size - - default_rrd_history_entries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "history", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); + // get default database update frequency - long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); - if(h != default_rrd_history_entries) { - config_set_number(CONFIG_SECTION_GLOBAL, "history", h); - default_rrd_history_entries = (int)h; + default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_DB, "update every", UPDATE_EVERY); + if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { + error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY); + default_rrd_update_every = UPDATE_EVERY; + config_set_number(CONFIG_SECTION_DB, "update every", default_rrd_update_every); } - if(default_rrd_history_entries < 5 || default_rrd_history_entries > RRD_HISTORY_ENTRIES_MAX) { - error("Invalid history entries %d given. Defaulting to %d.", default_rrd_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); - default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; + // ------------------------------------------------------------------------ + // get default memory mode for the database + + { + const char *mode = config_get(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode)); + default_rrd_memory_mode = rrd_memory_mode_id(mode); + if(strcmp(mode, rrd_memory_mode_name(default_rrd_memory_mode)) != 0) { + error("Invalid memory mode '%s' given. Using '%s'", mode, rrd_memory_mode_name(default_rrd_memory_mode)); + config_set(CONFIG_SECTION_DB, "mode", rrd_memory_mode_name(default_rrd_memory_mode)); + } } // ------------------------------------------------------------------------ - // get default database update frequency + // get default database size - default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_GLOBAL, "update every", UPDATE_EVERY); - if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { - error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY_MAX); - default_rrd_update_every = UPDATE_EVERY; + if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) { + default_rrd_history_entries = (int)config_get_number( + CONFIG_SECTION_DB, "retention", + align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); + + long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); + if (h != default_rrd_history_entries) { + config_set_number(CONFIG_SECTION_DB, "retention", h); + default_rrd_history_entries = (int)h; + } } // ------------------------------------------------------------------------ @@ -582,39 +646,38 @@ static void get_netdata_configured_variables() { netdata_configured_primary_plugins_dir = plugin_directories[PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH]; } - // ------------------------------------------------------------------------ - // get default memory mode for the database - - default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); #ifdef ENABLE_DBENGINE // ------------------------------------------------------------------------ // get default Database Engine page cache size in MiB - db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_GLOBAL, "page cache uses malloc", CONFIG_BOOLEAN_NO); - default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "page cache size", default_rrdeng_page_cache_mb); + db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "page cache with malloc", CONFIG_BOOLEAN_NO); + default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb); if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) { error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB); default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB; + config_set_number(CONFIG_SECTION_DB, "page cache size MB", default_rrdeng_page_cache_mb); } // ------------------------------------------------------------------------ // get default Database Engine disk space quota in MiB - default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine disk space", default_rrdeng_disk_quota_mb); + default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb); if(default_rrdeng_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB); default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB; + config_set_number(CONFIG_SECTION_DB, "dbengine disk space MB", default_rrdeng_disk_quota_mb); } - default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine multihost disk space", compute_multidb_diskspace()); + default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", compute_multidb_diskspace()); if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) { error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, default_rrdeng_disk_quota_mb); default_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb; + config_set_number(CONFIG_SECTION_DB, "dbengine multihost disk space MB", default_multidb_disk_quota_mb); } #else if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { - error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use memory mode ram instead."); - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM; + error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead."); + default_rrd_memory_mode = RRD_MEMORY_MODE_SAVE; } #endif // ------------------------------------------------------------------------ @@ -626,11 +689,12 @@ static void get_netdata_configured_variables() { // get KSM settings #ifdef MADV_MERGEABLE - enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm); + enable_ksm = config_get_boolean(CONFIG_SECTION_DB, "memory deduplication (ksm)", enable_ksm); #endif // -------------------------------------------------------------------- // metric correlations + enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations); default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method))); @@ -1232,39 +1296,47 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // get log filenames and settings + log_init(); error_log_limit_unlimited(); + // initialize the log files open_all_log_files(); #ifdef ENABLE_DBENGINE - default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine page fetch timeout", PAGE_CACHE_FETCH_WAIT_TIMEOUT); + default_rrdeng_page_fetch_timeout = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", PAGE_CACHE_FETCH_WAIT_TIMEOUT); if (default_rrdeng_page_fetch_timeout < 1) { - info("\"dbengine page fetch timeout\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_timeout); + info("'dbengine page fetch timeout secs' cannot be %d, using 1", default_rrdeng_page_fetch_timeout); default_rrdeng_page_fetch_timeout = 1; + config_set_number(CONFIG_SECTION_DB, "dbengine page fetch timeout secs", default_rrdeng_page_fetch_timeout); } - default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES); + default_rrdeng_page_fetch_retries = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page fetch retries", MAX_PAGE_CACHE_FETCH_RETRIES); if (default_rrdeng_page_fetch_retries < 1) { info("\"dbengine page fetch retries\" found in netdata.conf cannot be %d, using 1", default_rrdeng_page_fetch_retries); default_rrdeng_page_fetch_retries = 1; + config_set_number(CONFIG_SECTION_DB, "dbengine page fetch retries", default_rrdeng_page_fetch_retries); } #endif get_system_timezone(); + // -------------------------------------------------------------------- // get the certificate and start security + #ifdef ENABLE_HTTPS security_init(); #endif // -------------------------------------------------------------------- // This is the safest place to start the SILENCERS structure + set_silencers_filename(); health_initialize_global_silencers(); // -------------------------------------------------------------------- // Initialize ML configuration + ml_init(); // -------------------------------------------------------------------- @@ -1272,9 +1344,11 @@ int main(int argc, char **argv) { // block signals while initializing threads. // this causes the threads to block signals. + signals_block(); // setup the signals we want to use + signals_init(); // setup threads configs @@ -1303,6 +1377,7 @@ int main(int argc, char **argv) { if(web_server_mode != WEB_SERVER_MODE_NONE) api_listen_sockets_setup(); + } #ifdef NETDATA_INTERNAL_CHECKS @@ -1386,7 +1461,7 @@ int main(int argc, char **argv) { web_server_config_options(); - netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_GLOBAL, "enable zero metrics", CONFIG_BOOLEAN_NO); + netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_DB, "enable zero metrics", CONFIG_BOOLEAN_NO); set_late_global_environment(); diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 70fe388e46..ccaf44f9fd 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -4,7 +4,7 @@ static int check_number_printing(void) { struct { - calculated_number n; + NETDATA_DOUBLE n; const char *correct; } values[] = { { .n = 0, .correct = "0" }, @@ -22,8 +22,8 @@ static int check_number_printing(void) { char netdata[50], system[50]; int i, failed = 0; for(i = 0; values[i].correct ; i++) { - print_calculated_number(netdata, values[i].n); - snprintfz(system, 49, "%0.12" LONG_DOUBLE_MODIFIER, (LONG_DOUBLE)values[i].n); + print_netdata_double(netdata, values[i].n); + snprintfz(system, 49, "%0.12" NETDATA_DOUBLE_MODIFIER, (NETDATA_DOUBLE)values[i].n); int ok = 1; if(strcmp(netdata, values[i].correct) != 0) { @@ -95,35 +95,36 @@ static int check_rrdcalc_comparisons(void) { return 0; } -int check_storage_number(calculated_number n, int debug) { +int check_storage_number(NETDATA_DOUBLE n, int debug) { char buffer[100]; uint32_t flags = SN_DEFAULT_FLAGS; storage_number s = pack_storage_number(n, flags); - calculated_number d = unpack_storage_number(s); + NETDATA_DOUBLE d = unpack_storage_number(s); if(!does_storage_number_exist(s)) { - fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n); + fprintf(stderr, "Exists flags missing for number " NETDATA_DOUBLE_FORMAT "!\n", n); return 5; } - calculated_number ddiff = d - n; - calculated_number dcdiff = ddiff * 100.0 / n; + NETDATA_DOUBLE ddiff = d - n; + NETDATA_DOUBLE dcdiff = ddiff * 100.0 / n; if(dcdiff < 0) dcdiff = -dcdiff; - size_t len = (size_t)print_calculated_number(buffer, d); - calculated_number p = str2ld(buffer, NULL); - calculated_number pdiff = n - p; - calculated_number pcdiff = pdiff * 100.0 / n; + size_t len = (size_t)print_netdata_double(buffer, d); + NETDATA_DOUBLE p = str2ndd(buffer, NULL); + NETDATA_DOUBLE pdiff = n - p; + NETDATA_DOUBLE pcdiff = pdiff * 100.0 / n; if(pcdiff < 0) pcdiff = -pcdiff; if(debug) { fprintf(stderr, - CALCULATED_NUMBER_FORMAT " original\n" - CALCULATED_NUMBER_FORMAT " packed and unpacked, (stored as 0x%08X, diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n" - "%s printed after unpacked (%zu bytes)\n" - CALCULATED_NUMBER_FORMAT " re-parsed from printed (diff " CALCULATED_NUMBER_FORMAT ", " CALCULATED_NUMBER_FORMAT "%%)\n\n", + NETDATA_DOUBLE_FORMAT + " original\n" NETDATA_DOUBLE_FORMAT " packed and unpacked, (stored as 0x%08X, diff " NETDATA_DOUBLE_FORMAT + ", " NETDATA_DOUBLE_FORMAT "%%)\n" + "%s printed after unpacked (%zu bytes)\n" NETDATA_DOUBLE_FORMAT + " re-parsed from printed (diff " NETDATA_DOUBLE_FORMAT ", " NETDATA_DOUBLE_FORMAT "%%)\n\n", n, d, s, ddiff, dcdiff, buffer, len, @@ -132,10 +133,11 @@ int check_storage_number(calculated_number n, int debug) { if(len != strlen(buffer)) fprintf(stderr, "ERROR: printed number %s is reported to have length %zu but it has %zu\n", buffer, len, strlen(buffer)); if(dcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT) - fprintf(stderr, "WARNING: packing number " CALCULATED_NUMBER_FORMAT " has accuracy loss " CALCULATED_NUMBER_FORMAT " %%\n", n, dcdiff); + fprintf(stderr, "WARNING: packing number " NETDATA_DOUBLE_FORMAT " has accuracy loss " NETDATA_DOUBLE_FORMAT " %%\n", n, dcdiff); if(pcdiff > ACCURACY_LOSS_ACCEPTED_PERCENT) - fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " CALCULATED_NUMBER_FORMAT " has accuracy loss " CALCULATED_NUMBER_FORMAT " %%\n", n, pcdiff); + fprintf(stderr, "WARNING: re-parsing the packed, unpacked and printed number " NETDATA_DOUBLE_FORMAT + " has accuracy loss " NETDATA_DOUBLE_FORMAT " %%\n", n, pcdiff); } if(len != strlen(buffer)) return 1; @@ -144,8 +146,8 @@ int check_storage_number(calculated_number n, int debug) { return 0; } -calculated_number storage_number_min(calculated_number n) { - calculated_number r = 1, last; +NETDATA_DOUBLE storage_number_min(NETDATA_DOUBLE n) { + NETDATA_DOUBLE r = 1, last; do { last = n; @@ -159,12 +161,12 @@ calculated_number storage_number_min(calculated_number n) { void benchmark_storage_number(int loop, int multiplier) { int i, j; - calculated_number n, d; + NETDATA_DOUBLE n, d; storage_number s; unsigned long long user, system, total, mine, their; - calculated_number storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW); - calculated_number storage_number_positive_max = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW); + NETDATA_DOUBLE storage_number_positive_min = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW); + NETDATA_DOUBLE storage_number_positive_max = unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW); char buffer[100]; @@ -174,25 +176,25 @@ void benchmark_storage_number(int loop, int multiplier) { // ------------------------------------------------------------------------ - fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(calculated_number)); + fprintf(stderr, "SYSTEM LONG DOUBLE SIZE: %zu bytes\n", sizeof(NETDATA_DOUBLE)); fprintf(stderr, "NETDATA FLOATING POINT SIZE: %zu bytes\n", sizeof(storage_number)); - mine = (calculated_number)sizeof(storage_number) * (calculated_number)loop; - their = (calculated_number)sizeof(calculated_number) * (calculated_number)loop; + mine = (NETDATA_DOUBLE)sizeof(storage_number) * (NETDATA_DOUBLE)loop; + their = (NETDATA_DOUBLE)sizeof(NETDATA_DOUBLE) * (NETDATA_DOUBLE)loop; if(mine > their) { - fprintf(stderr, "\nNETDATA NEEDS %0.2" LONG_DOUBLE_MODIFIER " TIMES MORE MEMORY. Sorry!\n", (LONG_DOUBLE)(mine / their)); + fprintf(stderr, "\nNETDATA NEEDS %0.2" NETDATA_DOUBLE_MODIFIER " TIMES MORE MEMORY. Sorry!\n", (NETDATA_DOUBLE)(mine / their)); } else { - fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2" LONG_DOUBLE_MODIFIER " TIMES LESS MEMORY.\n", (LONG_DOUBLE)(their / mine)); + fprintf(stderr, "\nNETDATA INTERNAL FLOATING POINT ARITHMETICS NEEDS %0.2" NETDATA_DOUBLE_MODIFIER " TIMES LESS MEMORY.\n", (NETDATA_DOUBLE)(their / mine)); } fprintf(stderr, "\nNETDATA FLOATING POINT\n"); - fprintf(stderr, "MIN POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW)); - fprintf(stderr, "MAX POSITIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW)); - fprintf(stderr, "MIN NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MIN_RAW)); - fprintf(stderr, "MAX NEGATIVE VALUE " CALCULATED_NUMBER_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW)); - fprintf(stderr, "Maximum accuracy loss accepted: " CALCULATED_NUMBER_FORMAT "%%\n\n\n", (calculated_number)ACCURACY_LOSS_ACCEPTED_PERCENT); + fprintf(stderr, "MIN POSITIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MIN_RAW)); + fprintf(stderr, "MAX POSITIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_POSITIVE_MAX_RAW)); + fprintf(stderr, "MIN NEGATIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MIN_RAW)); + fprintf(stderr, "MAX NEGATIVE VALUE " NETDATA_DOUBLE_FORMAT "\n", unpack_storage_number(STORAGE_NUMBER_NEGATIVE_MAX_RAW)); + fprintf(stderr, "Maximum accuracy loss accepted: " NETDATA_DOUBLE_FORMAT "%%\n\n\n", (NETDATA_DOUBLE)ACCURACY_LOSS_ACCEPTED_PERCENT); // ------------------------------------------------------------------------ @@ -207,7 +209,7 @@ void benchmark_storage_number(int loop, int multiplier) { n *= multiplier; if(n > storage_number_positive_max) n = storage_number_positive_min; - print_calculated_number(buffer, n); + print_netdata_double(buffer, n); } } @@ -217,7 +219,8 @@ void benchmark_storage_number(int loop, int multiplier) { total = user + system; mine = total; - fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0)); + fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER + ", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0)); // ------------------------------------------------------------------------ @@ -231,7 +234,7 @@ void benchmark_storage_number(int loop, int multiplier) { for(i = 0; i < loop ;i++) { n *= multiplier; if(n > storage_number_positive_max) n = storage_number_positive_min; - snprintfz(buffer, 100, CALCULATED_NUMBER_FORMAT, n); + snprintfz(buffer, 100, NETDATA_DOUBLE_FORMAT, n); } } @@ -241,13 +244,14 @@ void benchmark_storage_number(int loop, int multiplier) { total = user + system; their = total; - fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER ", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0)); + fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER + ", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0)); if(mine > total) { - fprintf(stderr, "NETDATA CODE IS SLOWER %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(mine * 100.0 / their - 100.0)); + fprintf(stderr, "NETDATA CODE IS SLOWER %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(mine * 100.0 / their - 100.0)); } else { - fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(their * 100.0 / mine - 100.0)); + fprintf(stderr, "NETDATA CODE IS F A S T E R %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(their * 100.0 / mine - 100.0)); } // ------------------------------------------------------------------------ @@ -265,7 +269,7 @@ void benchmark_storage_number(int loop, int multiplier) { s = pack_storage_number(n, SN_DEFAULT_FLAGS); d = unpack_storage_number(s); - print_calculated_number(buffer, d); + print_netdata_double(buffer, d); } } @@ -275,13 +279,14 @@ void benchmark_storage_number(int loop, int multiplier) { total = user + system; mine = total; - fprintf(stderr, "user %0.5" LONG_DOUBLE_MODIFIER ", system %0.5" LONG_DOUBLE_MODIFIER ", total %0.5" LONG_DOUBLE_MODIFIER "\n", (LONG_DOUBLE)(user / 1000000.0), (LONG_DOUBLE)(system / 1000000.0), (LONG_DOUBLE)(total / 1000000.0)); + fprintf(stderr, "user %0.5" NETDATA_DOUBLE_MODIFIER ", system %0.5" NETDATA_DOUBLE_MODIFIER + ", total %0.5" NETDATA_DOUBLE_MODIFIER "\n", (NETDATA_DOUBLE)(user / 1000000.0), (NETDATA_DOUBLE)(system / 1000000.0), (NETDATA_DOUBLE)(total / 1000000.0)); if(mine > their) { - fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(mine * 100.0 / their - 100.0)); + fprintf(stderr, "WITH PACKING UNPACKING NETDATA CODE IS SLOWER %0.2" NETDATA_DOUBLE_MODIFIER " %%\n", (NETDATA_DOUBLE)(mine * 100.0 / their - 100.0)); } else { - fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2" LONG_DOUBLE_MODIFIER " %%\n", (LONG_DOUBLE)(their * 100.0 / mine - 100.0)); + fprintf(stderr, "EVEN WITH PACKING AND UNPACKING, NETDATA CODE IS F A S T E R %0.2" NETDATA_DOU