Database engine (#5282)

* Database engine prototype version 0

* Database engine initial integration with netdata POC

* Scalable database engine with file and memory management.

* Database engine integration with netdata

* Added MIN MAX definitions to fix alpine build of travis CI

* Bugfix for backends and new DB engine, remove useless rrdset_time2slot() calls and erroneous checks

* DB engine disk protocol correction

* Moved DB engine storage file location to /var/cache/netdata/{host}/dbengine

* Fix configure to require openSSL for DB engine

* Fix netdata daemon health not holding read lock when iterating chart dimensions

* Optimized query API for new DB engine and old netdata DB fallback code-path

* netdata database internal query API improvements and cleanup

* Bugfix for DB engine queries returning empty values

* Added netdata internal check for data queries for old and new DB

* Added statistics to DB engine and fixed memory corruption bug

* Added preliminary charts for DB engine statistics

* Changed DB engine ratio statistics to incremental

* Added netdata statistics charts for DB engine internal statistics

* Fix for netdata not compiling successfully when missing dbengine dependencies

* Added DB engine functional test to netdata unittest command parameter

* Implemented DB engine dataset generator based on example.random chart

* Fix build error in CI

* Support older versions of libuv1

* Fixes segmentation fault when using multiple DB engine instances concurrently

* Fix memory corruption bug

* Fixed createdataset advanced option not exiting

* Fix for DB engine not working on FreeBSD

* Support FreeBSD library paths of new dependencies

* Workaround for unsupported O_DIRECT in OS X

* Fix unittest crashing during cleanup

* Disable DB engine FS caching in Apple OS X since O_DIRECT is not available

* Fix segfault when unittest and DB engine dataset generator don't have permissions to create temporary host

* Modified DB engine dataset generator to create multiple files

* Toned down overzealous page cache prefetcher

* Reduce internal memory fragmentation for page-cache data pages

* Added documentation describing the DB engine

* Documentation bugfixes

* Fixed unit tests compilation errors since last rebase

* Added note to back-up the DB engine files in documentation

* Added codacy fix.

* Support old gcc versions for atomic counters in DB engine

40 files changed, 4823 insertions, 70 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 72a31ec4ef..870f7cf538 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -89,6 +89,46 @@ set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${ZLIB_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${ZLIB_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
 
+# -----------------------------------------------------------------------------
+# libuv multi-platform support library with a focus on asynchronous I/O
+
+pkg_check_modules(LIBUV REQUIRED libuv)
+set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBUV_CFLAGS_OTHER})
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBUV_LIBRARIES})
+set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBUV_INCLUDE_DIRS})
+
+# -----------------------------------------------------------------------------
+# lz4 Extremely Fast Compression algorithm
+
+pkg_check_modules(LIBLZ4 REQUIRED liblz4)
+set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER})
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES})
+set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS})
+
+# -----------------------------------------------------------------------------
+# Judy General purpose dynamic array
+
+# pkgconfig not working in Ubuntu, why? upstream package broken?
+#pkg_check_modules(JUDY REQUIRED Judy)
+#set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${JUDY_CFLAGS_OTHER})
+#set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${JUDY_LIBRARIES})
+#set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${JUDY_INCLUDE_DIRS})
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} "-lJudy")
+set(CMAKE_REQUIRED_LIBRARIES "Judy")
+check_symbol_exists("JudyLLast" "Judy.h" HAVE_JUDY)
+IF(HAVE_JUDY)
+    message(STATUS "Judy library found")
+ELSE()
+    message( FATAL_ERROR "libJudy required but not found. Try installing 'libjudy-dev' or 'Judy-devel'." )
+ENDIF()
+
+# -----------------------------------------------------------------------------
+# OpenSSL Cryptography and SSL/TLS Toolkit
+
+pkg_check_modules(OPENSSL REQUIRED openssl)
+set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${OPENSSL_CFLAGS_OTHER})
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${OPENSSL_LIBRARIES})
+set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIRS})
 
 # -----------------------------------------------------------------------------
 # Detect libcap
@@ -403,6 +443,19 @@ set(RRD_PLUGIN_FILES
         database/rrdsetvar.h
         database/rrdvar.c
         database/rrdvar.h
+        database/engine/rrdengine.c
+        database/engine/rrdengine.h
+        database/engine/rrddiskprotocol.h
+        database/engine/datafile.c
+        database/engine/datafile.h
+        database/engine/journalfile.c
+        database/engine/journalfile.h
+        database/engine/rrdenginelib.c
+        database/engine/rrdenginelib.h
+        database/engine/rrdengineapi.c
+        database/engine/rrdengineapi.h
+        database/engine/pagecache.c
+        database/engine/pagecache.h
         )
 
 set(WEB_PLUGIN_FILES
diff --git a/Makefile.am b/Makefile.am
index 7f3604ef83..df8194a251 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -311,6 +311,24 @@ RRD_PLUGIN_FILES = \
     database/rrdvar.h \
     $(NULL)
 
+if ENABLE_DBENGINE
+    RRD_PLUGIN_FILES += \
+        database/engine/rrdengine.c \
+        database/engine/rrdengine.h \
+        database/engine/rrddiskprotocol.h \
+        database/engine/datafile.c \
+        database/engine/datafile.h \
+        database/engine/journalfile.c \
+        database/engine/journalfile.h \
+        database/engine/rrdenginelib.c \
+        database/engine/rrdenginelib.h \
+        database/engine/rrdengineapi.c \
+        database/engine/rrdengineapi.h \
+        database/engine/pagecache.c \
+        database/engine/pagecache.h \
+        $(NULL)
+endif
+
 API_PLUGIN_FILES = \
     web/api/badges/web_buffer_svg.c \
     web/api/badges/web_buffer_svg.h \
@@ -477,6 +495,10 @@ NETDATA_COMMON_LIBS = \
 	$(OPTIONAL_MATH_LIBS) \
 	$(OPTIONAL_ZLIB_LIBS) \
 	$(OPTIONAL_UUID_LIBS) \
+	$(OPTIONAL_UV_LIBS) \
+	$(OPTIONAL_LZ4_LIBS) \
+	$(OPTIONAL_JUDY_LIBS) \
+	$(OPTIONAL_SSL_LIBS) \
 	$(NULL)
 # TODO: Find more graceful way to add libs for AWS Kinesis
 
diff --git a/backends/backends.c b/backends/backends.c
index deced30c46..5a8f3aa7b0 100644
--- a/backends/backends.c
+++ b/backends/backends.c
@@ -62,9 +62,11 @@ calculated_number backend_calculate_value_from_stored_data(
     (void)host;
 
     // find the edges of the rrd database for this chart
-    time_t first_t = rrdset_first_entry_t(st);
-    time_t last_t  = rrdset_last_entry_t(st);
+    time_t first_t = rd->state->query_ops.oldest_time(rd);
+    time_t last_t  = rd->state->query_ops.latest_time(rd);
     time_t update_every = st->update_every;
+    struct rrddim_query_handle handle;
+    storage_number n;
 
     // step back a little, to make sure we have complete data collection
     // for all metrics
@@ -105,6 +107,7 @@ calculated_number backend_calculate_value_from_stored_data(
     size_t counter = 0;
     calculated_number sum = 0;
 
+/*
     long    start_at_slot = rrdset_time2slot(st, before),
             stop_at_slot  = rrdset_time2slot(st, after),
             slot, stop_now = 0;
@@ -126,7 +129,21 @@ calculated_number backend_calculate_value_from_stored_data(
 
         counter++;
     }
+*/
+    for(rd->state->query_ops.init(rd, &handle, before, after) ; !rd->state->query_ops.is_finished(&handle) ; ) {
+        n = rd->state->query_ops.next_metric(&handle);
 
+        if(unlikely(!does_storage_number_exist(n))) {
+            // not collected
+            continue;
+        }
+
+        calculated_number value = unpack_storage_number(n);
+        sum += value;
+
+        counter++;
+    }
+    rd->state->query_ops.finalize(&handle);
     if(unlikely(!counter)) {
         debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu",
               host->hostname, st->id, rd->id,
diff --git a/configure.ac b/configure.ac
index 95b45dd42a..4e95858eda 100644
--- a/configure.ac
+++ b/configure.ac
@@ -131,6 +131,12 @@ AC_ARG_ENABLE(
     ,
     [enable_lto="detect"]
 )
+AC_ARG_ENABLE(
+    [dbengine],
+    [AS_HELP_STRING([--disable-dbengine], [disable netdata dbengine @<:@default autodetect@:>@])],
+    ,
+    [enable_dbengine="detect"]
+)
 
 
 # -----------------------------------------------------------------------------
@@ -188,7 +194,7 @@ case "$host_os" in
 freebsd*)
     build_target=freebsd
     build_target_id=2
-    CFLAGS="${CFLAGS} -I/usr/local/include"
+    CFLAGS="${CFLAGS} -I/usr/local/include -L/usr/local/lib"
     ;;
 darwin*)
     build_target=macos
@@ -243,6 +249,46 @@ AC_MSG_RESULT([${with_math}])
 
 
 # -----------------------------------------------------------------------------
+# libuv multi-platform support library with a focus on asynchronous I/O
+# TODO: check version, uv_fs_scandir_next only available in version >= 1.0
+
+AC_CHECK_LIB(
+    [uv],
+    [uv_fs_scandir_next],
+    [UV_LIBS="-luv"]
+)
+
+OPTIONAL_UV_CLFAGS="${UV_CFLAGS}"
+OPTIONAL_UV_LIBS="${UV_LIBS}"
+
+
+# -----------------------------------------------------------------------------
+# lz4 Extremely Fast Compression algorithm
+
+AC_CHECK_LIB(
+    [lz4],
+    [LZ4_decompress_safe],
+    [LZ4_LIBS="-llz4"]
+)
+
+OPTIONAL_LZ4_CLFAGS="${LZ4_CFLAGS}"
+OPTIONAL_LZ4_LIBS="${LZ4_LIBS}"
+
+
+# -----------------------------------------------------------------------------
+# Judy General purpose dynamic array
+
+AC_CHECK_LIB(
+    [Judy],
+    [JudyLIns],
+    [JUDY_LIBS="-lJudy"]
+)
+
+OPTIONAL_JUDY_CLFAGS="${JUDY_CFLAGS}"
+OPTIONAL_JUDY_LIBS="${JUDY_LIBS}"
+
+
+# -----------------------------------------------------------------------------
 # zlib
 
 PKG_CHECK_MODULES(
@@ -280,6 +326,43 @@ OPTIONAL_UUID_LIBS="${UUID_LIBS}"
 
 
 # -----------------------------------------------------------------------------
+# OpenSSL Cryptography and SSL/TLS Toolkit
+
+AC_CHECK_LIB(
+    [crypto],
+    [SHA256_Init],
+    [SSL_LIBS="-lcrypto -lssl"]
+)
+
+OPTIONAL_SSL_CLFAGS="${SSL_CFLAGS}"
+OPTIONAL_SSL_LIBS="${SSL_LIBS}"
+
+# -----------------------------------------------------------------------------
+# DB engine
+test "${enable_dbengine}" = "yes" -a -z "${UV_LIBS}" && \
+    AC_MSG_ERROR([libuv required but not found. Try installing 'libuv1-dev' or 'libuv-devel'.])
+
+test "${enable_dbengine}" = "yes" -a -z "${LZ4_LIBS}" && \
+    AC_MSG_ERROR([liblz4 required but not found. Try installing 'liblz4-dev' or 'lz4-devel'.])
+
+test "${enable_dbengine}" = "yes" -a -z "${JUDY_LIBS}" && \
+    AC_MSG_ERROR([libJudy required but not found. Try installing 'libjudy-dev' or 'Judy-devel'.])
+
+test "${enable_dbengine}" = "yes" -a -z "${SSL_LIBS}" && \
+    AC_MSG_ERROR([OpenSSL required but not found. Try installing 'libssl-dev' or 'openssl-devel'.])
+
+AC_MSG_CHECKING([if netdata dbengine should be used])
+if test "${enable_dbengine}" != "no" -a "${UV_LIBS}" -a "${LZ4_LIBS}" -a "${JUDY_LIBS}" -a "${SSL_LIBS}"; then
+    enable_dbengine="yes"
+    AC_DEFINE([ENABLE_DBENGINE], [1], [netdata dbengine usability])
+else
+    enable_dbengine="no"
+fi
+AC_MSG_RESULT([${enable_dbengine}])
+AM_CONDITIONAL([ENABLE_DBENGINE], [test "${enable_dbengine}" = "yes"])
+
+
+# -----------------------------------------------------------------------------
 # compiler options
 
 AC_ARG_VAR([SSE_CANDIDATE], [C compiler flags for SSE])
@@ -781,7 +864,12 @@ CPPFLAGS="\
 
 AC_SUBST([OPTIONAL_MATH_CFLAGS])
 AC_SUBST([OPTIONAL_MATH_LIBS])
-AC_SUBST([OPTIONAL_NFACCT_CFLAGS])
+AC_SUBST([OPTIONAL_RT_CLFAGS])
+AC_SUBST([OPTIONAL_UV_LIBS])
+AC_SUBST([OPTIONAL_LZ4_LIBS])
+AC_SUBST([OPTIONAL_JUDY_LIBS])
+AC_SUBST([OPTIONAL_SSL_LIBS])
+AC_SUBST([OPTIONAL_NFACCT_CLFAGS])
 AC_SUBST([OPTIONAL_NFACCT_LIBS])
 AC_SUBST([OPTIONAL_ZLIB_CFLAGS])
 AC_SUBST([OPTIONAL_ZLIB_LIBS])
@@ -831,6 +919,7 @@ AC_CONFIG_FILES([
     collectors/xenstat.plugin/Makefile
     daemon/Makefile
     database/Makefile
+    database/engine/Makefile
     diagrams/Makefile
     health/Makefile
     health/notifications/Makefile
diff --git a/daemon/README.md b/daemon/README.md
index 858394c770..62cc8c3b2b 100644
--- a/daemon/README.md
+++ b/daemon/README.md
@@ -164,6 +164,8 @@ The command line options of the netdata 1.10.0 version are the following:
 
   -W unittest              Run internal unittests and exit.
 
+  -W createdataset=N       Create a DB engine dataset of N seconds and exit.
+
   -W set section option value
                            set netdata.conf option from the command line.
 
diff --git a/daemon/config/README.md b/daemon/config/README.md
index 44bca2b2df..0508a19d99 100644
--- a/daemon/config/README.md
+++ b/daemon/config/README.md
@@ -57,7 +57,7 @@ cache directory | `/var/cache/netdata` | The directory the memory database will
 lib directory | `/var/lib/netdata` | Contains the alarm log and the netdata instance guid.
 home directory | `/var/cache/netdata` | Contains the db files for the collected metrics
 plugins directory | `"/usr/libexec/netdata/plugins.d" "/etc/netdata/custom-plugins.d"` | The directory plugin programs are kept. This setting supports multiple directories, space separated. If any directory path contains spaces, enclose it in single or double quotes.
-memory mode | `save` | When set to `save` netdata will save its round robin database on exit and load it on startup. When set to `map` the cache files will be updated in real time (check `man mmap` - do not set this on systems with heavy load or slow disks - the disks will continuously sync the in-memory database of netdata). When set to `ram` the round robin database will be temporary and it will be lost when netdata exits. `none` disables the database at this host. This also disables health monitoring (there cannot be health monitoring without a database). host access prefix | | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43).
+memory mode | `save` | When set to `save` netdata will save its round robin database on exit and load it on startup. When set to `map` the cache files will be updated in real time (check `man mmap` - do not set this on systems with heavy load or slow disks - the disks will continuously sync the in-memory database of netdata). When set to `dbengine` it behaves similarly to `map` but with much better disk and memory efficiency, however, with higher overhead. When set to `ram` the round robin database will be temporary and it will be lost when netdata exits. `none` disables the database at this host. This also disables health monitoring (there cannot be health monitoring without a database). host access prefix | | This is used in docker environments where /proc, /sys, etc have to be accessed via another path. You may also have to set SYS_PTRACE capability on the docker for this work. Check [issue 43](https://github.com/netdata/netdata/issues/43).
 memory deduplication (ksm) | `yes` | When set to `yes`, netdata will offer its in-memory round robin database to kernel same page merging (KSM) for deduplication. For more information check [Memory Deduplication - Kernel Same Page Merging - KSM](../../database/#ksm)
 TZ environment variable | `:/etc/localtime` | Where to find the timezone
 timezone | auto-detected | The timezone retrieved from the environment variable
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c
index 9933d0dfb2..9cc05abb7b 100644
--- a/daemon/global_statistics.c
+++ b/daemon/global_statistics.c
@@ -530,4 +530,223 @@ void global_statistics_charts(void) {
 
         rrdset_done(st_rrdr_points);
     }
+
+    // ----------------------------------------------------------------
+
+#ifdef ENABLE_DBENGINE
+    if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+        unsigned long long stats_array[27];
+
+        /* get localhost's DB engine's statistics */
+        rrdeng_get_27_statistics(localhost->rrdeng_ctx, stats_array);
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_compression = NULL;
+            static RRDDIM *rd_savings = NULL;
+
+            if (unlikely(!st_compression)) {
+                st_compression = rrdset_create_localhost(
+                        "netdata"
+                        , "dbengine_compression_ratio"
+                        , NULL
+                        , "dbengine"
+                        , NULL
+                        , "NetData DB engine data extents' compression savings ratio"
+                        , "percentage"
+                        , "netdata"
+                        , "stats"
+                        , 130502
+                        , localhost->rrd_update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+            }
+            else
+                rrdset_next(st_compression);
+
+            unsigned long long ratio;
+            unsigned long long compressed_content_size = stats_array[12];
+            unsigned long long content_size = stats_array[11];
+
+            if (content_size) {
+                // allow negative savings
+                ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
+            } else {
+                ratio = 0;
+            }
+            rrddim_set_by_pointer(st_compression, rd_savings, ratio);
+
+            rrdset_done(st_compression);
+        }
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_pg_cache_hit_ratio = NULL;
+            static RRDDIM *rd_hit_ratio = NULL;
+
+            if (unlikely(!st_pg_cache_hit_ratio)) {
+                st_pg_cache_hit_ratio = rrdset_create_localhost(
+                        "netdata"
+                        , "page_cache_hit_ratio"
+                        , NULL
+                        , "dbengine"
+                        , NULL
+                        , "NetData DB engine page cache hit ratio"
+                        , "percentage"
+                        , "netdata"
+                        , "stats"
+                        , 130503
+                        , localhost->rrd_update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rd_hit_ratio = rrddim_add(st_pg_cache_hit_ratio, "ratio", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
+            }
+            else
+                rrdset_next(st_pg_cache_hit_ratio);
+
+            static unsigned long long old_hits = 0;
+            static unsigned long long old_misses = 0;
+            unsigned long long hits = stats_array[7];
+            unsigned long long misses = stats_array[8];
+            unsigned long long hits_delta;
+            unsigned long long misses_delta;
+            unsigned long long ratio;
+
+            hits_delta = hits - old_hits;
+            misses_delta = misses - old_misses;
+            old_hits = hits;
+            old_misses = misses;
+
+            if (hits_delta + misses_delta) {
+                // allow negative savings
+                ratio = (hits_delta * 100 * 1000) / (hits_delta + misses_delta);
+            } else {
+                ratio = 0;
+            }
+            rrddim_set_by_pointer(st_pg_cache_hit_ratio, rd_hit_ratio, ratio);
+
+            rrdset_done(st_pg_cache_hit_ratio);
+        }
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_pg_cache_pages = NULL;
+            static RRDDIM *rd_populated = NULL;
+            static RRDDIM *rd_commited = NULL;
+            static RRDDIM *rd_insertions = NULL;
+            static RRDDIM *rd_deletions = NULL;
+            static RRDDIM *rd_backfills = NULL;
+            static RRDDIM *rd_evictions = NULL;
+
+            if (unlikely(!st_pg_cache_pages)) {
+                st_pg_cache_pages = rrdset_create_localhost(
+                        "netdata"
+                        , "page_cache_stats"
+                        , NULL
+                        , "dbengine"
+                        , NULL
+                        , "NetData DB engine page statistics"
+                        , "pages"
+                        , "netdata"
+                        , "stats"
+                        , 130504
+                        , localhost->rrd_update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+                rd_commited = rrddim_add(st_pg_cache_pages, "commited", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+                rd_insertions = rrddim_add(st_pg_cache_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+                rd_deletions = rrddim_add(st_pg_cache_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+                rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+                rd_evictions = rrddim_add(st_pg_cache_pages, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+            }
+            else
+                rrdset_next(st_pg_cache_pages);
+
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]);
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_commited, (collected_number)stats_array[4]);
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_insertions, (collected_number)stats_array[5]);
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_deletions, (collected_number)stats_array[6]);
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]);
+            rrddim_set_by_pointer(st_pg_cache_pages, rd_evictions, (collected_number)stats_array[10]);
+            rrdset_done(st_pg_cache_pages);
+        }
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_io_stats = NULL;
+            static RRDDIM *rd_reads = NULL;
+            static RRDDIM *rd_writes = NULL;
+
+            if (unlikely(!st_io_stats)) {
+                st_io_stats = rrdset_create_localhost(
+                "netdata"
+                , "dbengine_io_throughput"
+                , NULL
+                , "dbengine"
+                , NULL
+                , "NetData DB engine I/O throughput"
+                , "MiB/s"
+                , "netdata"
+                , "stats"
+                , 130505
+                , localhost->rrd_update_every
+                , RRDSET_TYPE_LINE
+                );
+
+                rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+                rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
+            }
+            else
+                rrdset_next(st_io_stats);
+
+            rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[17]);
+            rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[15]);
+            rrdset_done(st_io_stats);
+        }
+
+        // ----------------------------------------------------------------
+
+        {
+            static RRDSET *st_io_stats = NULL;
+            static RRDDIM *rd_reads = NULL;
+            static RRDDIM *rd_writes = NULL;
+
+            if (unlikely(!st_io_stats)) {
+                st_io_stats = rrdset_create_localhost(
+                        "netdata"
+                        , "dbengine_io_operations"
+                        , NULL
+                        , "dbengine"
+                        , NULL
+                        , "NetData DB engine I/O operations"
+                        , "operations/s"
+                        , "netdata"
+                        , "stats"
+                        , 130506
+                        , localhost->rrd_update_every
+                        , RRDSET_TYPE_LINE
+                );
+
+                rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
+                rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
+            }
+            else
+                rrdset_next(st_io_stats);
+
+            rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[18]);
+            rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]);
+            rrdset_done(st_io_stats);
+        }
+    }
+#endif
+
 }
diff --git a/daemon/main.c b/daemon/main.c
index fe9e9963c1..a1577fb937 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -301,6 +301,7 @@ int help(int exitcode) {
             "  -W stacksize=N           Set the stacksize (in bytes).\n\n"
             "  -W debug_flags=N         Set runtime tracing to debug.log.\n\n"
             "  -W unittest              Run internal unittests and exit.\n\n"
+            "  -W createdataset=N       Create a DB engine dataset of N seconds and exit.\n\n"
             "  -W set section option value\n"
             "                           set netdata.conf option from the command line.\n\n"
             "  -W simple-pattern pattern string\n"
@@ -471,6 +472,25 @@ static void get_netdata_configured_variables() {
 
     default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode)));
 
+#ifdef ENABLE_DBENGINE
+    // ------------------------------------------------------------------------
+    // get default Database Engine page cache size in MiB
+
+    default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "page cache size", default_rrdeng_page_cache_mb);
+    if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
+        error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
+        default_rrdeng_page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
+    }
+
+    // ------------------------------------------------------------------------
+    // get default Database Engine disk space quota in MiB
+
+    default_rrdeng_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "dbengine disk space", default_rrdeng_di