diff options
author | Markos Fountoulakis <44345837+mfundul@users.noreply.github.com> | 2019-05-15 08:28:06 +0300 |
---|---|---|
committer | Paul Emm. Katsoulakis <34388743+paulkatsoulakis@users.noreply.github.com> | 2019-05-15 08:28:06 +0300 |
commit | 6ca6d840dd19d5d7e9bacf93e011803ea5861496 (patch) | |
tree | f20393187806d642f94eab87f87180440089fb0a /daemon/global_statistics.c | |
parent | fed63b6e99dd70beb2cf9ccadd7c396aa05b2ae0 (diff) |
Database engine (#5282)
* Database engine prototype version 0
* Database engine initial integration with netdata POC
* Scalable database engine with file and memory management.
* Database engine integration with netdata
* Added MIN MAX definitions to fix alpine build of travis CI
* Bugfix for backends and new DB engine, remove useless rrdset_time2slot() calls and erroneous checks
* DB engine disk protocol correction
* Moved DB engine storage file location to /var/cache/netdata/{host}/dbengine
* Fix configure to require openSSL for DB engine
* Fix netdata daemon health not holding read lock when iterating chart dimensions
* Optimized query API for new DB engine and old netdata DB fallback code-path
* netdata database internal query API improvements and cleanup
* Bugfix for DB engine queries returning empty values
* Added netdata internal check for data queries for old and new DB
* Added statistics to DB engine and fixed memory corruption bug
* Added preliminary charts for DB engine statistics
* Changed DB engine ratio statistics to incremental
* Added netdata statistics charts for DB engine internal statistics
* Fix for netdata not compiling successfully when missing dbengine dependencies
* Added DB engine functional test to netdata unittest command parameter
* Implemented DB engine dataset generator based on example.random chart
* Fix build error in CI
* Support older versions of libuv1
* Fixes segmentation fault when using multiple DB engine instances concurrently
* Fix memory corruption bug
* Fixed createdataset advanced option not exiting
* Fix for DB engine not working on FreeBSD
* Support FreeBSD library paths of new dependencies
* Workaround for unsupported O_DIRECT in OS X
* Fix unittest crashing during cleanup
* Disable DB engine FS caching in Apple OS X since O_DIRECT is not available
* Fix segfault when unittest and DB engine dataset generator don't have permissions to create temporary host
* Modified DB engine dataset generator to create multiple files
* Toned down overzealous page cache prefetcher
* Reduce internal memory fragmentation for page-cache data pages
* Added documentation describing the DB engine
* Documentation bugfixes
* Fixed unit tests compilation errors since last rebase
* Added note to back-up the DB engine files in documentation
* Added codacy fix.
* Support old gcc versions for atomic counters in DB engine
Diffstat (limited to 'daemon/global_statistics.c')
-rw-r--r-- | daemon/global_statistics.c | 219 |
1 files changed, 219 insertions, 0 deletions
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 9933d0dfb2..9cc05abb7b 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -530,4 +530,223 @@ void global_statistics_charts(void) { rrdset_done(st_rrdr_points); } + + // ---------------------------------------------------------------- + +#ifdef ENABLE_DBENGINE + if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) { + unsigned long long stats_array[27]; + + /* get localhost's DB engine's statistics */ + rrdeng_get_27_statistics(localhost->rrdeng_ctx, stats_array); + + // ---------------------------------------------------------------- + + { + static RRDSET *st_compression = NULL; + static RRDDIM *rd_savings = NULL; + + if (unlikely(!st_compression)) { + st_compression = rrdset_create_localhost( + "netdata" + , "dbengine_compression_ratio" + , NULL + , "dbengine" + , NULL + , "NetData DB engine data extents' compression savings ratio" + , "percentage" + , "netdata" + , "stats" + , 130502 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_compression); + + unsigned long long ratio; + unsigned long long compressed_content_size = stats_array[12]; + unsigned long long content_size = stats_array[11]; + + if (content_size) { + // allow negative savings + ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size; + } else { + ratio = 0; + } + rrddim_set_by_pointer(st_compression, rd_savings, ratio); + + rrdset_done(st_compression); + } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_pg_cache_hit_ratio = NULL; + static RRDDIM *rd_hit_ratio = NULL; + + if (unlikely(!st_pg_cache_hit_ratio)) { + st_pg_cache_hit_ratio = rrdset_create_localhost( + "netdata" + , "page_cache_hit_ratio" + , NULL + , "dbengine" + , NULL + , "NetData DB engine page cache hit ratio" + , "percentage" + , "netdata" + , "stats" + , 130503 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_hit_ratio = rrddim_add(st_pg_cache_hit_ratio, "ratio", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE); + } + else + rrdset_next(st_pg_cache_hit_ratio); + + static unsigned long long old_hits = 0; + static unsigned long long old_misses = 0; + unsigned long long hits = stats_array[7]; + unsigned long long misses = stats_array[8]; + unsigned long long hits_delta; + unsigned long long misses_delta; + unsigned long long ratio; + + hits_delta = hits - old_hits; + misses_delta = misses - old_misses; + old_hits = hits; + old_misses = misses; + + if (hits_delta + misses_delta) { + // allow negative savings + ratio = (hits_delta * 100 * 1000) / (hits_delta + misses_delta); + } else { + ratio = 0; + } + rrddim_set_by_pointer(st_pg_cache_hit_ratio, rd_hit_ratio, ratio); + + rrdset_done(st_pg_cache_hit_ratio); + } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_pg_cache_pages = NULL; + static RRDDIM *rd_populated = NULL; + static RRDDIM *rd_commited = NULL; + static RRDDIM *rd_insertions = NULL; + static RRDDIM *rd_deletions = NULL; + static RRDDIM *rd_backfills = NULL; + static RRDDIM *rd_evictions = NULL; + + if (unlikely(!st_pg_cache_pages)) { + st_pg_cache_pages = rrdset_create_localhost( + "netdata" + , "page_cache_stats" + , NULL + , "dbengine" + , NULL + , "NetData DB engine page statistics" + , "pages" + , "netdata" + , "stats" + , 130504 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_commited = rrddim_add(st_pg_cache_pages, "commited", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + rd_insertions = rrddim_add(st_pg_cache_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_deletions = rrddim_add(st_pg_cache_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_evictions = rrddim_add(st_pg_cache_pages, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_pg_cache_pages); + + rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_commited, (collected_number)stats_array[4]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_insertions, (collected_number)stats_array[5]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_deletions, (collected_number)stats_array[6]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]); + rrddim_set_by_pointer(st_pg_cache_pages, rd_evictions, (collected_number)stats_array[10]); + rrdset_done(st_pg_cache_pages); + } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_io_stats = NULL; + static RRDDIM *rd_reads = NULL; + static RRDDIM *rd_writes = NULL; + + if (unlikely(!st_io_stats)) { + st_io_stats = rrdset_create_localhost( + "netdata" + , "dbengine_io_throughput" + , NULL + , "dbengine" + , NULL + , "NetData DB engine I/O throughput" + , "MiB/s" + , "netdata" + , "stats" + , 130505 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); + rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_io_stats); + + rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[17]); + rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[15]); + rrdset_done(st_io_stats); + } + + // ---------------------------------------------------------------- + + { + static RRDSET *st_io_stats = NULL; + static RRDDIM *rd_reads = NULL; + static RRDDIM *rd_writes = NULL; + + if (unlikely(!st_io_stats)) { + st_io_stats = rrdset_create_localhost( + "netdata" + , "dbengine_io_operations" + , NULL + , "dbengine" + , NULL + , "NetData DB engine I/O operations" + , "operations/s" + , "netdata" + , "stats" + , 130506 + , localhost->rrd_update_every + , RRDSET_TYPE_LINE + ); + + rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); + rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); + } + else + rrdset_next(st_io_stats); + + rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[18]); + rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]); + rrdset_done(st_io_stats); + } + } +#endif + } |