Refactor RRD code. (#15423)

* Storage engine.

* Host indexes to rrdb

* Move globals to rrdb

* Move storage_tiers_backfill to rrdb

* default_rrd_update_every to rrdb

* default_rrd_history_entries to rrdb

* gap_when_lost_iterations_above to rrdb

* rrdset_free_obsolete_time_s to rrdb

* libuv_worker_threads to rrdb

* ieee754_doubles to rrdb

* rrdhost_free_orphan_time_s to rrdb

* rrd_rwlock to rrdb

* localhost to rrdb

* rm extern from func decls

* mv rrd macro under rrd.h

* default_rrdeng_page_cache_mb to rrdb

* default_rrdeng_extent_cache_mb to rrdb

* db_engine_journal_check to rrdb

* default_rrdeng_disk_quota_mb to rrdb

* default_multidb_disk_quota_mb to rrdb

* multidb_ctx to rrdb

* page_type_size to rrdb

* tier_page_size to rrdb

* No storage_engine_id in rrdim functions

* storage_engine_id is provided by st

* Update to fix merge conflict.

* Update field name

* Remove unnecessary macros from rrd.h

* Rm unused type decls

* Rm duplicate func decls

* make internal function static

* Make the rest of public dbengine funcs accept a storage_instance.

* No more rrdengine_instance :)

* rm rrdset_debug from rrd.h

* Use rrdb to access globals in ML and ACLK

Missed due to not having the submodules in the
worktree.

* rm total_number

* rm RRDVAR_TYPE_TOTAL

* rm unused inline

* Rm names from typedef'd enums

* rm unused header include

* Move include

* Rm unused header include

* s/rrdhost_find_or_create/rrdhost_get_or_create/g

* s/find_host_by_node_id/rrdhost_find_by_node_id/

Also, remove duplicate definition in rrdcontext.c

* rm macro used only once

* rm macro used only once

* Reduce rrd.h api by moving funcs into a collector specific utils header

* Remove unused func

* Move parser specific function out of rrd.h

* return storage_number instead of void pointer

* move code related to rrd initialization out of rrdhost.c

* Remove tier_grouping from rrdim_tier

Saves 8 * storage_tiers bytes per dimension.

* Fix rebase

* s/rrd_update_every/update_every/

* Mark functions as static and constify args

* Add license notes and file to build systems.

* Remove remaining non-log/config mentions of memory mode

* Move rrdlabels api to separate file.

Also, move localhost functions that loads
labels outside of database/ and into daemon/

* Remove function decl in rrd.h

* merge rrdhost_cache_dir_for_rrdset_alloc into rrdset_cache_dir

* Do not expose internal function from rrd.h

* Rm NETDATA_RRD_INTERNALS

Only one function decl is covered. We have more
database internal functions that we currently
expose for no good reason. These will be placed
in a separate internal header in follow up PRs.

* Add license note

* Include libnetdata.h instead of aral.h

* Use rrdb to access localhost

* Fix builds without dbengine

* Add header to build system files

* Add rrdlabels.h to build systems

* Move func def from rrd.h to rrdhost.c

* Fix macos build

* Rm non-existing function

* Rebase master

* Define buffer length macro in ad_charts.

* Fix FreeBSD builds.

* Mark functions static

* Rm func decls without definitions

* Rebase master

* Rebase master

* Properly initialize value of storage tiers.

* Fix build after rebase.

15 files changed, 27 insertions, 5 deletions

diff --git a/libnetdata/aral/aral.c b/libnetdata/aral/aral.c
index 16328db69f..cc7173e5d4 100644
--- a/libnetdata/aral/aral.c
+++ b/libnetdata/aral/aral.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "../libnetdata.h"
-#include "aral.h"
 
 #ifdef NETDATA_TRACE_ALLOCATIONS
 #define TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS , const char *file, const char *function, size_t line
diff --git a/libnetdata/aral/aral.h b/libnetdata/aral/aral.h
index 96f5a9c44f..048bbfdf09 100644
--- a/libnetdata/aral/aral.h
+++ b/libnetdata/aral/aral.h
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
 
 #ifndef ARAL_H
 #define ARAL_H 1
diff --git a/libnetdata/circular_buffer/circular_buffer.c b/libnetdata/circular_buffer/circular_buffer.c
index 7ffe6b8bc7..b89ae58aea 100644
--- a/libnetdata/circular_buffer/circular_buffer.c
+++ b/libnetdata/circular_buffer/circular_buffer.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "../libnetdata.h"
 
 struct circular_buffer *cbuffer_new(size_t initial, size_t max, size_t *statistics) {
diff --git a/libnetdata/circular_buffer/circular_buffer.h b/libnetdata/circular_buffer/circular_buffer.h
index 9d29a84d70..76e60d34c3 100644
--- a/libnetdata/circular_buffer/circular_buffer.h
+++ b/libnetdata/circular_buffer/circular_buffer.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #ifndef CIRCULAR_BUFFER_H
 #define CIRCULAR_BUFFER_H 1
 
diff --git a/libnetdata/json/jsmn.c b/libnetdata/json/jsmn.c
index 2f48bd65a1..7866ddf0cc 100644
--- a/libnetdata/json/jsmn.c
+++ b/libnetdata/json/jsmn.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include <stdlib.h>
 
 #include "jsmn.h"
diff --git a/libnetdata/json/jsmn.h b/libnetdata/json/jsmn.h
index beff586c66..546832a6b6 100644
--- a/libnetdata/json/jsmn.h
+++ b/libnetdata/json/jsmn.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #ifndef __JSMN_H_
 #define __JSMN_H_
 
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
index 272ba8f17a..4d7b15f30b 100644
--- a/libnetdata/libnetdata.c
+++ b/libnetdata/libnetdata.c
@@ -1139,8 +1139,8 @@ void *netdata_mmap(const char *filename, size_t size, int flags, int ksm, bool r
 {
     // netdata_log_info("netdata_mmap('%s', %zu", filename, size);
 
-    // MAP_SHARED is used in memory mode map
-    // MAP_PRIVATE is used in memory mode ram and save
+    // MAP_SHARED is used in storage engine MAP
+    // MAP_PRIVATE is used in storage engine RAM and SAVE
 
     if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE)))
         fatal("Neither MAP_SHARED or MAP_PRIVATE were given to netdata_mmap()");
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
index 8b8c7206e4..66b6746943 100644
--- a/libnetdata/libnetdata.h
+++ b/libnetdata/libnetdata.h
@@ -248,6 +248,7 @@ size_t judy_aral_structures(void);
 #define ABS(x) (((x) < 0)? (-(x)) : (x))
 #define MIN(a,b) (((a)<(b))?(a):(b))
 #define MAX(a,b) (((a)>(b))?(a):(b))
+#define CLAMP(value, min, max) MIN(MAX((value), (min)), (max))
 
 #define GUID_LEN 36
 
@@ -840,8 +841,6 @@ extern char *netdata_configured_host_prefix;
 // BEWARE: this exists in alarm-notify.sh
 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
 
-#define RRD_STORAGE_TIERS 5
-
 static inline size_t struct_natural_alignment(size_t size) __attribute__((const));
 
 #define STRUCT_NATURAL_ALIGNMENT (sizeof(uintptr_t) * 2)
diff --git a/libnetdata/onewayalloc/onewayalloc.c b/libnetdata/onewayalloc/onewayalloc.c
index 05c9f2a9dd..901388b664 100644
--- a/libnetdata/onewayalloc/onewayalloc.c
+++ b/libnetdata/onewayalloc/onewayalloc.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "onewayalloc.h"
 
 // https://www.gnu.org/software/libc/manual/html_node/Aligned-Memory-Blocks.html
diff --git a/libnetdata/onewayalloc/onewayalloc.h b/libnetdata/onewayalloc/onewayalloc.h
index a415b063b4..3d3e08b31d 100644
--- a/libnetdata/onewayalloc/onewayalloc.h
+++ b/libnetdata/onewayalloc/onewayalloc.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #ifndef ONEWAYALLOC_H
 #define ONEWAYALLOC_H 1
 
diff --git a/libnetdata/socket/security.c b/libnetdata/socket/security.c
index c1bb763456..8dc865b5f7 100644
--- a/libnetdata/socket/security.c
+++ b/libnetdata/socket/security.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "../libnetdata.h"
 
 #ifdef ENABLE_HTTPS
diff --git a/libnetdata/socket/security.h b/libnetdata/socket/security.h
index a4082a0776..e7dd765aa9 100644
--- a/libnetdata/socket/security.h
+++ b/libnetdata/socket/security.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #ifndef NETDATA_SECURITY_H
 # define NETDATA_SECURITY_H
 
diff --git a/libnetdata/string/string.h b/libnetdata/string/string.h
index 70840ee9a2..5082211a92 100644
--- a/libnetdata/string/string.h
+++ b/libnetdata/string/string.h
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
 
 #ifndef NETDATA_STRING_H
 #define NETDATA_STRING_H 1
diff --git a/libnetdata/worker_utilization/worker_utilization.c b/libnetdata/worker_utilization/worker_utilization.c
index ad45dbc7f1..227d61bc91 100644
--- a/libnetdata/worker_utilization/worker_utilization.c
+++ b/libnetdata/worker_utilization/worker_utilization.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #include "worker_utilization.h"
 
 #define WORKER_IDLE 'I'
diff --git a/libnetdata/worker_utilization/worker_utilization.h b/libnetdata/worker_utilization/worker_utilization.h
index 6745a010bc..e94731b3ce 100644
--- a/libnetdata/worker_utilization/worker_utilization.h
+++ b/libnetdata/worker_utilization/worker_utilization.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
 #ifndef WORKER_UTILIZATION_H
 #define WORKER_UTILIZATION_H 1