From 6ca6d840dd19d5d7e9bacf93e011803ea5861496 Mon Sep 17 00:00:00 2001 From: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com> Date: Wed, 15 May 2019 08:28:06 +0300 Subject: Database engine (#5282) * Database engine prototype version 0 * Database engine initial integration with netdata POC * Scalable database engine with file and memory management. * Database engine integration with netdata * Added MIN MAX definitions to fix alpine build of travis CI * Bugfix for backends and new DB engine, remove useless rrdset_time2slot() calls and erroneous checks * DB engine disk protocol correction * Moved DB engine storage file location to /var/cache/netdata/{host}/dbengine * Fix configure to require openSSL for DB engine * Fix netdata daemon health not holding read lock when iterating chart dimensions * Optimized query API for new DB engine and old netdata DB fallback code-path * netdata database internal query API improvements and cleanup * Bugfix for DB engine queries returning empty values * Added netdata internal check for data queries for old and new DB * Added statistics to DB engine and fixed memory corruption bug * Added preliminary charts for DB engine statistics * Changed DB engine ratio statistics to incremental * Added netdata statistics charts for DB engine internal statistics * Fix for netdata not compiling successfully when missing dbengine dependencies * Added DB engine functional test to netdata unittest command parameter * Implemented DB engine dataset generator based on example.random chart * Fix build error in CI * Support older versions of libuv1 * Fixes segmentation fault when using multiple DB engine instances concurrently * Fix memory corruption bug * Fixed createdataset advanced option not exiting * Fix for DB engine not working on FreeBSD * Support FreeBSD library paths of new dependencies * Workaround for unsupported O_DIRECT in OS X * Fix unittest crashing during cleanup * Disable DB engine FS caching in Apple OS X since O_DIRECT is not available * Fix segfault when unittest and DB engine dataset generator don't have permissions to create temporary host * Modified DB engine dataset generator to create multiple files * Toned down overzealous page cache prefetcher * Reduce internal memory fragmentation for page-cache data pages * Added documentation describing the DB engine * Documentation bugfixes * Fixed unit tests compilation errors since last rebase * Added note to back-up the DB engine files in documentation * Added codacy fix. * Support old gcc versions for atomic counters in DB engine --- daemon/unit_test.c | 212 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) (limited to 'daemon/unit_test.c') diff --git a/daemon/unit_test.c b/daemon/unit_test.c index a92a50a111..f9b58ce6b8 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1566,3 +1566,215 @@ int unit_test(long delay, long shift) return ret; } + +#ifdef ENABLE_DBENGINE +static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number value, time_t now) +{ + rd->last_collected_time.tv_sec = now; + rd->last_collected_time.tv_usec = 0; + rd->collected_value = value; + rd->updated = 1; + + rd->collections_counter++; + + collected_number v = (value >= 0) ? value : -value; + if(unlikely(v > rd->collected_value_max)) rd->collected_value_max = v; +} + +int test_dbengine(void) +{ + const int CHARTS = 128; + const int DIMS = 16; /* That gives us 2048 metrics */ + const int POINTS = 16384; /* This produces 128MiB of metric data */ + const int QUERY_BATCH = 4096; + uint8_t same; + int i, j, k, c, errors; + RRDHOST *host = NULL; + RRDSET *st[CHARTS]; + RRDDIM *rd[CHARTS][DIMS]; + char name[101]; + time_t time_now; + collected_number last; + struct rrddim_query_handle handle; + calculated_number value, expected; + storage_number n; + + error_log_limit_unlimited(); + fprintf(stderr, "\nRunning DB-engine test\n"); + + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + + debug(D_RRDHOST, "Initializing localhost with hostname 'unittest-dbengine'"); + host = rrdhost_find_or_create( + "unittest-dbengine" + , "unittest-dbengine" + , "unittest-dbengine" + , os_type + , netdata_configured_timezone + , config_get(CONFIG_SECTION_BACKEND, "host tags", "") + , program_name + , program_version + , default_rrd_update_every + , default_rrd_history_entries + , RRD_MEMORY_MODE_DBENGINE + , default_health_enabled + , default_rrdpush_enabled + , default_rrdpush_destination + , default_rrdpush_api_key + , default_rrdpush_send_charts_matching + , NULL + ); + if (NULL == host) + return 1; + + for (i = 0 ; i < CHARTS ; ++i) { + snprintfz(name, 100, "dbengine-chart-%d", i); + + // create the chart + st[i] = rrdset_create(host, "netdata", name, name, "netdata", NULL, "Unit Testing", "a value", "unittest", + NULL, 1, 1, RRDSET_TYPE_LINE); + rrdset_flag_set(st[i], RRDSET_FLAG_DEBUG); + rrdset_flag_set(st[i], RRDSET_FLAG_STORE_FIRST); + for (j = 0 ; j < DIMS ; ++j) { + snprintfz(name, 100, "dim-%d", j); + + rd[i][j] = rrddim_add(st[i], name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + } + + // feed it with the test data + time_now = 1; + last = 0; + for (i = 0 ; i < CHARTS ; ++i) { + for (j = 0 ; j < DIMS ; ++j) { + rd[i][j]->last_collected_time.tv_sec = + st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now; + rd[i][j]->last_collected_time.tv_usec = + st[i]->last_collected_time.tv_usec = st[i]->last_updated.tv_usec = 0; + } + } + for(c = 0; c < POINTS ; ++c) { + ++time_now; // time_now = c + 2 + for (i = 0 ; i < CHARTS ; ++i) { + st[i]->usec_since_last_update = USEC_PER_SEC; + + for (j = 0; j < DIMS; ++j) { + last = i * DIMS * POINTS + j * POINTS + c; + rrddim_set_by_pointer_fake_time(rd[i][j], last, time_now); + } + rrdset_done(st[i]); + } + } + + // check the result + errors = 0; + + for(c = 0; c < POINTS ; c += QUERY_BATCH) { + time_now = c + 2; + for (i = 0 ; i < CHARTS ; ++i) { + for (j = 0; j < DIMS; ++j) { + rd[i][j]->state->query_ops.init(rd[i][j], &handle, time_now, time_now + QUERY_BATCH); + for (k = 0; k < QUERY_BATCH; ++k) { + last = i * DIMS * POINTS + j * POINTS + c + k; + expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + + n = rd[i][j]->state->query_ops.next_metric(&handle); + value = unpack_storage_number(n); + + same = (calculated_number_round(value * 10000000.0) == calculated_number_round(expected * 10000000.0)) ? 1 : 0; + if(!same) { + fprintf(stderr, " DB-engine unittest %s/%s: at %lu secs, expecting value " + CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", ### E R R O R ###\n", + st[i]->name, rd[i][j]->name, (unsigned long)time_now + k, expected, value); + errors++; + } + } + rd[i][j]->state->query_ops.finalize(&handle); + } + } + } + + rrdeng_exit(host->rrdeng_ctx); + rrd_wrlock(); + rrdhost_delete_charts(host); + rrd_unlock(); + + return errors; +} + +void generate_dbengine_dataset(unsigned history_seconds) +{ + const int DIMS = 128; + const uint64_t EXPECTED_COMPRESSION_RATIO = 94; + int j; + RRDHOST *host = NULL; + RRDSET *st; + RRDDIM *rd[DIMS]; + char name[101]; + time_t time_current, time_present; + + default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + default_rrdeng_page_cache_mb = 128; + /* Worst case for uncompressible data */ + default_rrdeng_disk_quota_mb = (((uint64_t)DIMS) * sizeof(storage_number) * history_seconds) / (1024 * 1024); + default_rrdeng_disk_quota_mb -= default_rrdeng_disk_quota_mb * EXPECTED_COMPRESSION_RATIO / 100; + + error_log_limit_unlimited(); + debug(D_RRDHOST, "Initializing localhost with hostname 'dbengine-dataset'"); + + host = rrdhost_find_or_create( + "dbengine-dataset" + , "dbengine-dataset" + , "dbengine-dataset" + , os_type + , netdata_configured_timezone + , config_get(CONFIG_SECTION_BACKEND, "host tags", "") + , program_name + , program_version + , default_rrd_update_every + , default_rrd_history_entries + , RRD_MEMORY_MODE_DBENGINE + , default_health_enabled + , default_rrdpush_enabled + , default_rrdpush_destination + , default_rrdpush_api_key + , default_rrdpush_send_charts_matching + , NULL + ); + if (NULL == host) + return; + + fprintf(stderr, "\nRunning DB-engine workload generator\n"); + + // create the chart + st = rrdset_create(host, "example", "random", "random", "example", NULL, "random", "random", "random", + NULL, 1, 1, RRDSET_TYPE_LINE); + for (j = 0 ; j < DIMS ; ++j) { + snprintfz(name, 100, "random%d", j); + + rd[j] = rrddim_add(st, name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); + } + + time_present = now_realtime_sec(); + // feed it with the test data + time_current = time_present - history_seconds; + for (j = 0 ; j < DIMS ; ++j) { + rd[j]->last_collected_time.tv_sec = + st->last_collected_time.tv_sec = st->last_updated.tv_sec = time_current; + rd[j]->last_collected_time.tv_usec = + st->last_collected_time.tv_usec = st->last_updated.tv_usec = 0; + } + for( ; time_current < time_present; ++time_current) { + st->usec_since_last_update = USEC_PER_SEC; + + for (j = 0; j < DIMS; ++j) { + rrddim_set_by_pointer_fake_time(rd[j], (time_current + j) % 128, time_current); + } + rrdset_done(st); + } + rrd_wrlock(); + rrdhost_free(host); + rrd_unlock(); + +} +#endif \ No newline at end of file -- cgit v1.2.3