Implemented default disk space size calculation for multihost db (#9504)

Implemented default disk space size calculation for multihost db (#9504)

6 files changed, 94 insertions, 2 deletions

diff --git a/daemon/main.c b/daemon/main.c
index 158237406c..4df623c84e 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -557,6 +557,13 @@ static void get_netdata_configured_variables() {
         error("Invalid dbengine disk space %d given. Defaulting to %d.", default_rrdeng_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
         default_rrdeng_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
     }
+
+    default_multidb_disk_quota_mb = (int) config_get_number(CONFIG_SECTION_GLOBAL, "multidb disk space", compute_multidb_diskspace());
+    if(default_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB) {
+        error("Invalid multidb disk space %d given. Defaulting to %d.", default_multidb_disk_quota_mb, RRDENG_MIN_DISK_SPACE_MB);
+        default_multidb_disk_quota_mb = RRDENG_MIN_DISK_SPACE_MB;
+    }
+
 #endif
     // ------------------------------------------------------------------------
 
diff --git a/database/engine/metadata_log/metadatalog.c b/database/engine/metadata_log/metadatalog.c
index e163ff8bcc..8be938f3d5 100644
--- a/database/engine/metadata_log/metadatalog.c
+++ b/database/engine/metadata_log/metadatalog.c
@@ -414,4 +414,4 @@ void error_with_guid(uuid_t *uuid, char *reason)
     uuid_unparse_lower(*uuid, uuid_str);
     errno = 0;
     error("%s (GUID = %s)", reason, uuid_str);
-}
\ No newline at end of file
+}
diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
index 3b9a24df87..315d4ad51d 100755
--- a/database/engine/rrdengineapi.c
+++ b/database/engine/rrdengineapi.c
@@ -6,6 +6,7 @@ static struct rrdengine_instance default_global_ctx;
 
 int default_rrdeng_page_cache_mb = 32;
 int default_rrdeng_disk_quota_mb = 256;
+int default_multidb_disk_quota_mb = 256;
 /* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
 uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
 
diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h
index 42382717ea..17d6e2aceb 100644
--- a/database/engine/rrdengineapi.h
+++ b/database/engine/rrdengineapi.h
@@ -14,6 +14,7 @@
 
 extern int default_rrdeng_page_cache_mb;
 extern int default_rrdeng_disk_quota_mb;
+extern int default_multidb_disk_quota_mb;
 extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;
 
 struct rrdeng_region_info {
@@ -58,4 +59,4 @@ extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *db
 extern int rrdeng_exit(struct rrdengine_instance *ctx);
 extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
 
-#endif /* NETDATA_RRDENGINEAPI_H */
\ No newline at end of file
+#endif /* NETDATA_RRDENGINEAPI_H */
diff --git a/database/engine/rrdenginelib.c b/database/engine/rrdenginelib.c
index 2619e8f5f4..19a115c54b 100644
--- a/database/engine/rrdenginelib.c
+++ b/database/engine/rrdenginelib.c
@@ -208,3 +208,84 @@ char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t si
     );
     return str;
 }
+
+int is_legacy_child(const char *machine_guid)
+{
+    uuid_t uuid;
+    char  dbengine_file[FILENAME_MAX+1];
+
+    if (!uuid_parse(machine_guid, uuid)) {
+        uv_fs_t stat_req;
+        snprintfz(dbengine_file, FILENAME_MAX, "%s/%s/dbengine", netdata_configured_cache_dir, machine_guid);
+        int rc = uv_fs_stat(NULL, &stat_req, dbengine_file, NULL);
+        if (likely(rc == 0 && ((stat_req.statbuf.st_mode & S_IFMT) == S_IFDIR))) {
+            //info("Found legacy engine folder \"%s\"", dbengine_file);
+            return 1;
+        }
+    }
+    return 0;
+}
+
+int count_legacy_children(char *dbfiles_path)
+{
+    int ret;
+    uv_fs_t req;
+    uv_dirent_t dent;
+    int legacy_engines = 0;
+
+    ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
+    if (ret < 0) {
+        uv_fs_req_cleanup(&req);
+        error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
+        return ret;
+    }
+
+    while(UV_EOF != uv_fs_scandir_next(&req, &dent)) {
+        if (dent.type == UV_DIRENT_DIR) {
+            if (is_legacy_child(dent.name))
+                legacy_engines++;
+        }
+    }
+    uv_fs_req_cleanup(&req);
+    return legacy_engines;
+}
+
+int compute_multidb_diskspace()
+{
+    char multidb_disk_space_file[FILENAME_MAX + 1];
+    FILE *fp;
+    int computed_multidb_disk_quota_mb = -1;
+
+    snprintfz(multidb_disk_space_file, FILENAME_MAX, "%s/dbengine_multihost_size", netdata_configured_varlib_dir);
+    fp = fopen(multidb_disk_space_file, "r");
+    if (likely(fp)) {
+        int rc = fscanf(fp, "%d", &computed_multidb_disk_quota_mb);
+        fclose(fp);
+        if (unlikely(rc != 1 || computed_multidb_disk_quota_mb < RRDENG_MIN_DISK_SPACE_MB)) {
+            errno = 0;
+            error("File '%s' contains invalid input, it will be rebuild", multidb_disk_space_file);
+            computed_multidb_disk_quota_mb = -1;
+        }
+    }
+
+    if (computed_multidb_disk_quota_mb == -1) {
+        int rc = count_legacy_children(netdata_configured_cache_dir);
+        if (likely(rc >= 0)) {
+            computed_multidb_disk_quota_mb = (rc + 1) * default_rrdeng_disk_quota_mb;
+            //info("Found %d legacy dbengines, setting multidb diskspace to %dMB", rc, computed_multidb_disk_quota_mb);
+
+            // TODO: will activate the next block of code when multidb is in place
+//            fp = fopen(multidb_disk_space_file, "w");
+//            if (likely(fp)) {
+//                fprintf(fp, "%d", rc * default_rrdeng_disk_quota_mb);
+//                info("Created file '%s' to store the computed value", multidb_disk_space_file);
+//                fclose(fp);
+//            } else
+//                error("Failed to store the default multidb disk quota size on '%s'", multidb_disk_space_file);
+        }
+        else
+            computed_multidb_disk_quota_mb = default_rrdeng_disk_quota_mb;
+    }
+
+    return computed_multidb_disk_quota_mb;
+}
diff --git a/database/engine/rrdenginelib.h b/database/engine/rrdenginelib.h
index 982085d295..1184358f63 100644
--- a/database/engine/rrdenginelib.h
+++ b/database/engine/rrdenginelib.h
@@ -110,5 +110,7 @@ static inline int open_file_buffered_io(char *path, int flags, uv_file *file)
     return open_file_for_io(path, flags, file, 0);
 }
 extern char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t size);
+extern int compute_multidb_diskspace();
+extern int is_legacy_child(const char *machine_guid);
 
 #endif /* NETDATA_RRDENGINELIB_H */
\ No newline at end of file