From 9a6fd6366fafc5585b3fe9d2be2258df4738050d Mon Sep 17 00:00:00 2001 From: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> Date: Thu, 28 Jul 2022 21:53:05 +0300 Subject: Store host label information in the metadata database (#13441) * Create new host_label table * Add generic function to store chart and host labels Add function to store host labels Cleanup old host labels * Store labels from localhost and children (streaming) Remove the host label info if the host is deleted * Delete host labels before insert --- database/rrd.h | 2 + database/rrdhost.c | 1 + database/sqlite/sqlite_functions.c | 112 +++++++++++++++++++++++++++---------- database/sqlite/sqlite_functions.h | 1 + 4 files changed, 88 insertions(+), 28 deletions(-) (limited to 'database') diff --git a/database/rrd.h b/database/rrd.h index fbb870b5fc..5ef6c4b782 100644 --- a/database/rrd.h +++ b/database/rrd.h @@ -209,6 +209,8 @@ typedef enum rrdlabel_source { RRDLABEL_FLAG_NEW = (1 << 31) // marks for rrdlabels internal use - they are not exposed outside rrdlabels } RRDLABEL_SRC; +#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT) + extern DICTIONARY *rrdlabels_create(void); extern void rrdlabels_destroy(DICTIONARY *labels_dict); extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls); diff --git a/database/rrdhost.c b/database/rrdhost.c index c34eac0606..ec1d450d88 100644 --- a/database/rrdhost.c +++ b/database/rrdhost.c @@ -1362,6 +1362,7 @@ void reload_host_labels(void) { rrdhost_load_auto_labels(); rrdlabels_remove_all_unmarked(localhost->host_labels); + sql_store_host_labels(localhost); health_label_log_save(localhost); diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index 7ea95cb231..b55272a9b9 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -42,6 +42,9 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, " "date_created INT, PRIMARY KEY(host_id, system_key));", + "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, " + "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key));", + "CREATE TABLE IF NOT EXISTS chart_hash_map(chart_id blob , hash_id blob, UNIQUE (chart_id, hash_id));", "CREATE TABLE IF NOT EXISTS chart_hash(hash_id blob PRIMARY KEY,type text, id text, name text, " @@ -74,6 +77,7 @@ const char *database_cleanup[] = { "DELETE FROM chart_hash WHERE hash_id NOT IN (SELECT hash_id FROM chart_hash_map);", "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);", "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);", + "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host);", NULL }; @@ -764,7 +768,7 @@ static int exec_statement_with_uuid(const char *sql, uuid_t *uuid) rc = execute_insert(res); if (likely(rc == SQLITE_DONE)) - result = 0; + result = SQLITE_OK; else error_report("Failed to execute %s, rc = %d", sql, rc); @@ -1599,6 +1603,45 @@ void add_migrated_file(char *path, uint64_t file_size) return; } +static int sql_store_label(sqlite3_stmt *res, uuid_t *uuid, int source_type, const char *label, const char *value) +{ + int rc; + + rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind UUID parameter to store label information"); + goto skip_store; + } + + rc = sqlite3_bind_int(res, 2, source_type); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind type parameter to store label information"); + goto skip_store; + } + + rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind label parameter to store label information"); + goto skip_store; + } + + rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind value parameter to store label information"); + goto skip_store; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store label entry, rc = %d", rc); + +skip_store: + if (unlikely(sqlite3_reset(res) != SQLITE_OK)) + error_report("Failed to reset the prepared statement when storing label information"); + + return rc != SQLITE_DONE; +} + #define SQL_INS_CHART_LABEL "insert or replace into chart_label " \ "(chart_id, source_type, label_key, label_value, date_created) " \ "values (@chart, @source, @label, @value, unixepoch());" @@ -1622,39 +1665,35 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha } } - rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind chart_id parameter to store label information"); - goto failed; - } + sql_store_label(res, chart_uuid, source_type, label, value); - rc = sqlite3_bind_int(res, 2, source_type); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind type parameter to store label information"); - goto failed; - } + return; +} - rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind label parameter to store label information"); - goto failed; - } +#define SQL_INS_HOST_LABEL "INSERT OR REPLACE INTO host_label " \ + "(host_id, source_type, label_key, label_value, date_created) " \ + "values (@chart, @source, @label, @value, unixepoch());" - rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind value parameter to store label information"); - goto failed; - } +static void sql_store_host_label(uuid_t *host_uuid, int source_type, const char *label, const char *value) +{ + static __thread sqlite3_stmt *res = NULL; + int rc; - rc = execute_insert(res); - if (unlikely(rc != SQLITE_DONE)) - error_report("Failed to store chart label entry, rc = %d", rc); + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } -failed: - if (unlikely(sqlite3_reset(res) != SQLITE_OK)) - error_report("Failed to reset the prepared statement when storing chart label information"); + if (unlikely(!res)) { + rc = prepare_statement(db_meta, SQL_INS_HOST_LABEL, &res); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement store chart labels"); + return; + } + } - return; + (void) sql_store_label(res, host_uuid, source_type, label, value); } int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id, @@ -2601,6 +2640,23 @@ void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_inf return; } +static int save_host_label_callback(const char *name, const char *value, RRDLABEL_SRC label_source, void *data) +{ + RRDHOST *host = (RRDHOST *)data; + sql_store_host_label(&host->host_uuid, (int)label_source & ~(RRDLABEL_FLAG_INTERNAL), name, value); + return 0; +} + +#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid;" +void sql_store_host_labels(RRDHOST *host) +{ + int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid); + if (rc != SQLITE_OK) + error_report("Failed to remove old host labels for host %s", host->hostname); + + rrdlabels_walkthrough_read(host->host_labels, save_host_label_callback, host); +} + // Utils int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null) { diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index a02b73241a..7b1a88e293 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -111,4 +111,5 @@ int init_database_batch(sqlite3 *database, int rebuild, int init_type, const cha void migrate_localhost(uuid_t *host_uuid); extern void sql_store_host_system_info(uuid_t *host_id, const struct rrdhost_system_info *system_info); extern void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info); +void sql_store_host_labels(RRDHOST *host); #endif //NETDATA_SQLITE_FUNCTIONS_H -- cgit v1.2.3