diff options
| author | Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> | 2023-11-28 20:41:05 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-11-28 20:41:05 +0200 |
| commit | d6a81a99279f9a2caccdc7d88bb6e50382c55f1b (patch) | |
| tree | ffb50b1f3cbba2b71015309705728eb0a102626f | |
| parent | fcb9513b912c91e1616e2bef81687b5f40e77a3c (diff) | |
Fix occasional shutdown deadlock (#16495)
* Wait for RRDENG_OPCODE_CTX_QUIESCE to complete before attempting rrd_finalize_collection_for_all_hosts
* Submit RRDENG_OPCODE_CTX_QUIESCE for all tiers and then wait for completion
| -rw-r--r-- | daemon/main.c | 7 | ||||
| -rw-r--r-- | database/engine/rrdengine.c | 3 |
2 files changed, 7 insertions, 3 deletions
diff --git a/daemon/main.c b/daemon/main.c index d7b15a7baa..1b8c556a74 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -424,6 +424,13 @@ void netdata_cleanup_and_exit(int ret) { delta_shutdown_time("flush dbengine tiers"); for (size_t tier = 0; tier < storage_tiers; tier++) rrdeng_prepare_exit(multidb_ctx[tier]); + + for (size_t tier = 0; tier < storage_tiers; tier++) { + if (!multidb_ctx[tier]) + continue; + completion_wait_for(&multidb_ctx[tier]->quiesce.completion); + completion_destroy(&multidb_ctx[tier]->quiesce.completion); + } } #endif diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c index 5cbbb4a870..3ae270252b 100644 --- a/database/engine/rrdengine.c +++ b/database/engine/rrdengine.c @@ -1345,9 +1345,6 @@ static void after_ctx_shutdown(struct rrdengine_instance *ctx __maybe_unused, vo static void *ctx_shutdown_tp_worker(struct rrdengine_instance *ctx __maybe_unused, void *data __maybe_unused, struct completion *completion __maybe_unused, uv_work_t *uv_work_req __maybe_unused) { worker_is_busy(UV_EVENT_DBENGINE_SHUTDOWN); - completion_wait_for(&ctx->quiesce.completion); - completion_destroy(&ctx->quiesce.completion); - bool logged = false; while(__atomic_load_n(&ctx->atomic.extents_currently_being_flushed, __ATOMIC_RELAXED) || __atomic_load_n(&ctx->atomic.inflight_queries, __ATOMIC_RELAXED)) { |