Improve agent shutdown (#16959)

* Clear detection thread on stop
Handle multi db close

* cancel_threads should be called once

5 files changed, 13 insertions, 2 deletions

diff --git a/src/daemon/main.c b/src/daemon/main.c
index 652f05f61d..3684260710 100644
--- a/src/daemon/main.c
+++ b/src/daemon/main.c
@@ -691,6 +691,9 @@ static void set_nofile_limit(struct rlimit *rl) {
 void cancel_main_threads() {
     nd_log_limits_unlimited();
 
+    if (!static_threads)
+        return;
+
     int i, found = 0;
     usec_t max = 5 * USEC_PER_SEC, step = 100000;
     for (i = 0; static_threads[i].name != NULL ; i++) {
@@ -732,6 +735,7 @@ void cancel_main_threads() {
         freez(static_threads[i].thread);
 
     freez(static_threads);
+    static_threads = NULL;
 }
 
 static const struct option_def {
diff --git a/src/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c
index 5fad94be2f..683ed12cd3 100644
--- a/src/database/sqlite/sqlite_context.c
+++ b/src/database/sqlite/sqlite_context.c
@@ -86,6 +86,7 @@ void sql_close_context_database(void)
     rc = sqlite3_close_v2(db_context_meta);
     if (unlikely(rc != SQLITE_OK))
         error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc));
+    db_context_meta = NULL;
 }
 
 //
diff --git a/src/database/sqlite/sqlite_functions.c b/src/database/sqlite/sqlite_functions.c
index c1be331081..fab3847c27 100644
--- a/src/database/sqlite/sqlite_functions.c
+++ b/src/database/sqlite/sqlite_functions.c
@@ -518,6 +518,7 @@ void sql_close_database(void)
     rc = sqlite3_close_v2(db_meta);
     if (unlikely(rc != SQLITE_OK))
         error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));
+    db_meta = NULL;
 }
 
 int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
diff --git a/src/database/sqlite/sqlite_metadata.c b/src/database/sqlite/sqlite_metadata.c
index 6cb9bdf9cc..82fd2b22b4 100644
--- a/src/database/sqlite/sqlite_metadata.c
+++ b/src/database/sqlite/sqlite_metadata.c
@@ -1826,9 +1826,12 @@ void metadata_sync_shutdown(void)
 
 void metadata_sync_shutdown_prepare(void)
 {
-    if (unlikely(!metasync_worker.loop))
+    static bool running = false;
+    if (unlikely(!metasync_worker.loop || running))
         return;
 
+    running = true;
+
     struct metadata_cmd cmd;
     memset(&cmd, 0, sizeof(cmd));
 
diff --git a/src/ml/ml.cc b/src/ml/ml.cc
index 46476cd8f8..5844b0414d 100644
--- a/src/ml/ml.cc
+++ b/src/ml/ml.cc
@@ -1805,12 +1805,13 @@ void ml_init()
 }
 
 void ml_fini() {
-    if (!Cfg.enable_anomaly_detection)
+    if (!Cfg.enable_anomaly_detection || !db)
         return;
 
     int rc = sqlite3_close_v2(db);
     if (unlikely(rc != SQLITE_OK))
         error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));
+    db = NULL;
 }
 
 void ml_start_threads() {
@@ -1845,6 +1846,7 @@ void ml_stop_threads()
         return;
 
     netdata_thread_join(Cfg.detection_thread, NULL);
+    Cfg.detection_thread = 0;
 
     // signal the training queue of each thread
     for (size_t idx = 0; idx != Cfg.num_training_threads; idx++) {