Detect stored metric size by page type (#13334)

* Report unknown page only once
Get metric storage size by the page type
Verify validity of the page and skip problematic ones

* Change PAGE_SIZE to PAGE_POINT_SIZE_BYTES

* Add bitmap256 and unittests

* Fix unit test
tier_page_type array
page_type_size arrays

* Add another counter to not rely on uint8_t overflow to stop the test loop

3 files changed, 110 insertions, 1 deletions

diff --git a/daemon/main.c b/daemon/main.c
index eec5473808..ece125bcb4 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -973,7 +973,8 @@ int main(int argc, char **argv) {
                                 return 1;
                             if (unit_test_str2ld())
                                 return 1;
-
+                            if (unit_test_bitmap256())
+                                return 1;
                             // No call to load the config file on this code-path
                             post_conf_load(&user);
                             get_netdata_configured_variables();
diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index 0cfe5abb5f..f7f4bb24d0 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -1606,6 +1606,113 @@ error:
     return 1;
 }
 
+int unit_test_bitmap256(void) {
+    fprintf(stderr, "%s() running...\n", __FUNCTION__ );
+
+    BITMAP256 test_bitmap = {0};
+
+    bitmap256_set_bit(&test_bitmap, 0, 1);
+    bitmap256_set_bit(&test_bitmap, 64, 1);
+    bitmap256_set_bit(&test_bitmap, 128, 1);
+    bitmap256_set_bit(&test_bitmap, 192, 1);
+    if (test_bitmap.data[0] == 1)
+        fprintf(stderr, "%s() INDEX 1 is OK\n", __FUNCTION__ );
+    if (test_bitmap.data[1] == 1)
+        fprintf(stderr, "%s() INDEX 65 is OK\n", __FUNCTION__ );
+    if (test_bitmap.data[2] == 1)
+        fprintf(stderr, "%s() INDEX 129 is OK\n", __FUNCTION__ );
+    if (test_bitmap.data[3] == 1)
+        fprintf(stderr, "%s() INDEX 192 is OK\n", __FUNCTION__ );
+
+    uint8_t i=0;
+    int j = 0;
+    do {
+        bitmap256_set_bit(&test_bitmap, i++, 1);
+        j++;
+    } while (j < 256);
+
+    if (test_bitmap.data[0] == 0xffffffff)
+        fprintf(stderr, "%s() INDEX 0 is fully set OK\n", __FUNCTION__);
+    if (test_bitmap.data[1] == 0xffffffff)
+        fprintf(stderr, "%s() INDEX 1 is fully set OK\n", __FUNCTION__);
+    if (test_bitmap.data[2] == 0xffffffff)
+        fprintf(stderr, "%s() INDEX 2 is fully set OK\n", __FUNCTION__);
+    if (test_bitmap.data[3] == 0xffffffff)
+        fprintf(stderr, "%s() INDEX 3 is fully set OK\n", __FUNCTION__);
+
+    i = 0;
+    j = 0;
+    do {
+        bitmap256_set_bit(&test_bitmap, i++, 0);
+        j++;
+    } while (j < 256);
+
+    if (test_bitmap.data[0] == 0)
+        fprintf(stderr, "%s() INDEX 0 is reset OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 0 is not reset FAILED\n", __FUNCTION__);
+        return 1;
+    }
+    if (test_bitmap.data[1] == 0)
+        fprintf(stderr, "%s() INDEX 1 is reset OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 1 is not reset FAILED\n", __FUNCTION__);
+        return 1;
+    }
+
+    if (test_bitmap.data[2] == 0)
+        fprintf(stderr, "%s() INDEX 2 is reset OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 2 is not reset FAILED\n", __FUNCTION__);
+        return 1;
+    }
+
+    if (test_bitmap.data[3] == 0)
+        fprintf(stderr, "%s() INDEX 3 is reset OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 3 is not reset FAILED\n", __FUNCTION__);
+        return 1;
+    }
+
+    i=0;
+    j = 0;
+    do {
+        bitmap256_set_bit(&test_bitmap, i, 1);
+        i += 4;
+        j += 4;
+    } while (j < 256);
+
+    if (test_bitmap.data[0] == 0x11111111)
+        fprintf(stderr, "%s() INDEX 0 is 0x11111111 set OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 0 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[0]);
+        return 1;
+    }
+
+    if (test_bitmap.data[1] == 0x11111111)
+        fprintf(stderr, "%s() INDEX 1 is 0x11111111 set OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 1 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[1]);
+        return 1;
+    }
+
+    if (test_bitmap.data[2] == 0x11111111)
+        fprintf(stderr, "%s() INDEX 2 is 0x11111111 set OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 2 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[2]);
+        return 1;
+    }
+
+    if (test_bitmap.data[3] == 0x11111111)
+        fprintf(stderr, "%s() INDEX 3 is 0x11111111 set OK\n", __FUNCTION__);
+    else {
+        fprintf(stderr, "%s() INDEX 3 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[3]);
+        return 1;
+    }
+
+    fprintf(stderr, "%s() tests passed\n", __FUNCTION__);
+    return 0;
+}
 
 #ifdef ENABLE_DBENGINE
 static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number value, time_t now)
diff --git a/daemon/unit_test.h b/daemon/unit_test.h
index 6a7a966c3e..2d2533afed 100644
--- a/daemon/unit_test.h
+++ b/daemon/unit_test.h
@@ -10,6 +10,7 @@ extern int unit_test_str2ld(void);
 extern int unit_test_buffer(void);
 extern int unit_test_static_threads(void);
 extern int test_sqlite(void);
+extern int unit_test_bitmap256(void);
 #ifdef ENABLE_DBENGINE
 extern int test_dbengine(void);
 extern void generate_dbengine_dataset(unsigned history_seconds);