diff options
author | Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> | 2022-10-16 23:15:14 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-16 23:15:14 +0300 |
commit | 08cab7222478d06481c070b7f5027ead1c5d2b01 (patch) | |
tree | a7089ad0e88f6b14601a5de888c3699dd0af859e /daemon | |
parent | 70dad7e1263d066319ad13f13ea0c5b39cf664f5 (diff) |
Add a thread to asynchronously process metadata updates (#13783)
* Remove old metalog text fle processing
* Add metadata event loop
* Move functions from sqlite_functions.c to sqlite_metadata.c
Queue updates to the metadata event loop
Migration to remove unused tables
Cleanup unused functions
* Queue chart labels to metadata
* Store chart labels to metadata
* During shutdown, run full speed
* Add shutdown prepare
Handle SHUTDOWN in the cmd queue function
Add worker thread to handle host/chart/dimension metadata doing dictionary traversals
* Remove unused RRDIM_FLAG_ACLK
Add flags to trigger host/chart/dimension metadata processing
* Incremental processing of chart metadata writes
* Store host labels
* Remove redundant return statements
* Change unit tests / cleanup
* Fix rescheduling
* Schedule chart labels update by setting the RRDSET_FLAG_METADATA_UPDATE flag
* Queue commands to update metadata for dimension and host labels
* Make sure we do a final scan to store metadata during shutdown (if needed)
* Remove unused structures
Adjust queue size since we do batch processing of updates without queueing individual messages
Remove pragma mmap for now
Fix memory leak during sqlite unittest (minor)
* Dont update if we are in archive mode
* Cleanup
* Build entire message payload and store
* Initialize worker completion properly
* Properly skip host check for pending metadata updates
* Report bind param failures
Add worker request inside the data payload
Initialize variables to silence warnings
Rebase on master
* Report the chart id (not the dimension) and the dimension id when storing a dimension
* Compilation warnings in 32bit
* Add DEFINE for the queries
* Remove commented out code
* * Remove items parameter from unitest
* Remove commented out code
* sqlite_metadata.h contains only public items
* Use sleep_usec instead of usleep
* Rename metadata_database_init_cmd_queue to metadata_init_cmd_queue
* Rename metadata_database_enq_cmd_noblock to metadata_enq_cmd_noblock
Diffstat (limited to 'daemon')
-rw-r--r-- | daemon/global_statistics.c | 1 | ||||
-rw-r--r-- | daemon/main.c | 5 | ||||
-rw-r--r-- | daemon/service.c | 2 | ||||
-rw-r--r-- | daemon/unit_test.c | 2 |
4 files changed, 9 insertions, 1 deletions
diff --git a/daemon/global_statistics.c b/daemon/global_statistics.c index 62997cb785..32b885738c 100644 --- a/daemon/global_statistics.c +++ b/daemon/global_statistics.c @@ -1832,6 +1832,7 @@ static struct worker_utilization all_workers_utilization[] = { { .name = "WEB", .family = "workers web server", .priority = 1000000 }, { .name = "ACLKQUERY", .family = "workers aclk query", .priority = 1000000 }, { .name = "ACLKSYNC", .family = "workers aclk host sync", .priority = 1000000 }, + { .name = "METASYNC", .family = "workers metadata sync", .priority = 1000000 }, { .name = "PLUGINSD", .family = "workers plugins.d", .priority = 1000000 }, { .name = "STATSD", .family = "workers plugin statsd", .priority = 1000000 }, { .name = "STATSDFLUSH", .family = "workers plugin statsd flush", .priority = 1000000 }, diff --git a/daemon/main.c b/daemon/main.c index df93bfa25b..2bd8be8a48 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -60,7 +60,9 @@ void netdata_cleanup_and_exit(int ret) { rrdeng_prepare_exit(multidb_ctx[tier]); } #endif + metadata_sync_shutdown_prepare(); rrdhost_free_all(); + metadata_sync_shutdown(); #ifdef ENABLE_DBENGINE if(dbengine_enabled) { for (int tier = 0; tier < storage_tiers; tier++) @@ -1039,6 +1041,9 @@ int main(int argc, char **argv) { else if(strcmp(optarg, "rrdlabelstest") == 0) { return rrdlabels_unittest(); } + else if(strcmp(optarg, "metatest") == 0) { + return metadata_unittest(); + } else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) { optarg += strlen(createdataset_string); unsigned history_seconds = strtoul(optarg, NULL, 0); diff --git a/daemon/service.c b/daemon/service.c index 3a267402d1..a45019813a 100644 --- a/daemon/service.c +++ b/daemon/service.c @@ -60,7 +60,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) { if (tiers_available == tiers_said_yes && tiers_said_yes) { /* This metric has no data and no references */ - delete_dimension_uuid(&rd->metric_uuid); + metaqueue_delete_dimension_uuid(&rd->metric_uuid); } else { /* Do not delete this dimension */ diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 4d61c18984..32ebb2996c 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1564,8 +1564,10 @@ int test_sqlite(void) { buffer_free(sql); fprintf(stderr,"SQLite is OK\n"); + rc = sqlite3_close_v2(db_meta); return 0; error: + rc = sqlite3_close_v2(db_meta); fprintf(stderr,"SQLite statement failed: %s\n", buffer_tostring(sql)); buffer_free(sql); fprintf(stderr,"SQLite tests failed\n"); |