diff options
author | vkalintiris <vasilis@netdata.cloud> | 2024-02-26 11:16:24 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-26 11:16:24 +0200 |
commit | 5f68469ee56bff4f9f10d9a94c4461e76a7b2c52 (patch) | |
tree | a3b677a798d7fe4d64492070af5876969337616e | |
parent | 433dc480f9d9efdcf64fc9ed4b382baf0d4d55f2 (diff) |
Add watcher thread to report shutdown steps. (#17010)
* Add watcher thread to report shutdown steps.
* Add license header
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rw-r--r-- | src/daemon/main.c | 169 | ||||
-rw-r--r-- | src/daemon/static_threads.c | 2 | ||||
-rw-r--r-- | src/daemon/watcher.c | 138 | ||||
-rw-r--r-- | src/daemon/watcher.h | 55 |
5 files changed, 261 insertions, 105 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 7808f4312b..320a294e84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -824,6 +824,8 @@ set(DAEMON_FILES src/daemon/signals.c src/daemon/signals.h src/daemon/service.c + src/daemon/watcher.c + src/daemon/watcher.h src/daemon/static_threads.c src/daemon/static_threads.h src/daemon/commands.c diff --git a/src/daemon/main.c b/src/daemon/main.c index d11e3a61e3..46f008cb56 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -2,6 +2,7 @@ #include "common.h" #include "buildinfo.h" +#include "daemon/watcher.h" #include "static_threads.h" #include "database/engine/page_test.h" @@ -299,25 +300,10 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) { return (running == 0); } -#define delta_shutdown_time(msg) \ - do { \ - usec_t now_ut = now_monotonic_usec(); \ - if(prev_msg) \ - netdata_log_info("NETDATA SHUTDOWN: in %llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \ - else \ - netdata_log_info("NETDATA SHUTDOWN: next: %s", msg); \ - last_ut = now_ut; \ - prev_msg = msg; \ - timeout = false; \ - } while(0) - void web_client_cache_destroy(void); void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) { - usec_t started_ut = now_monotonic_usec(); - usec_t last_ut = started_ut; - const char *prev_msg = NULL; - bool timeout = false; + watcher_shutdown_begin(); nd_log_limits_unlimited(); netdata_log_info("NETDATA SHUTDOWN: initializing shutdown with code %d...", ret); @@ -330,104 +316,78 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re statistic = (analytics_statistic_t) {"EXIT", ret?"ERROR":"OK","-"}; analytics_statistic_send(&statistic); - delta_shutdown_time("create shutdown file"); - char agent_crash_file[FILENAME_MAX + 1]; char agent_incomplete_shutdown_file[FILENAME_MAX + 1]; snprintfz(agent_crash_file, FILENAME_MAX, "%s/.agent_crash", netdata_configured_varlib_dir); snprintfz(agent_incomplete_shutdown_file, FILENAME_MAX, "%s/.agent_incomplete_shutdown", netdata_configured_varlib_dir); (void) rename(agent_crash_file, agent_incomplete_shutdown_file); + watcher_step_complete(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE); #ifdef ENABLE_DBENGINE if(dbengine_enabled) { - delta_shutdown_time("dbengine exit mode"); for (size_t tier = 0; tier < storage_tiers; tier++) rrdeng_exit_mode(multidb_ctx[tier]); } #endif - - delta_shutdown_time("close webrtc connections"); + watcher_step_complete(WATCHER_STEP_ID_DBENGINE_EXIT_MODE); webrtc_close_all_connections(); + watcher_step_complete(WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS); - delta_shutdown_time("disable maintenance, new queries, new web requests, new streaming connections and aclk"); + service_signal_exit(SERVICE_MAINTENANCE | ABILITY_DATA_QUERIES | ABILITY_WEB_REQUESTS | + ABILITY_STREAMING_CONNECTIONS | SERVICE_ACLK | SERVICE_ACLKSYNC); + watcher_step_complete(WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK); - service_signal_exit( - SERVICE_MAINTENANCE - | ABILITY_DATA_QUERIES - | ABILITY_WEB_REQUESTS - | ABILITY_STREAMING_CONNECTIONS - | SERVICE_ACLK - | SERVICE_ACLKSYNC - ); + service_wait_exit(SERVICE_MAINTENANCE, 3 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD); - delta_shutdown_time("stop maintenance thread"); + service_wait_exit(SERVICE_EXPORTERS | SERVICE_HEALTH | SERVICE_WEB_SERVER | SERVICE_HTTPD, 3 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS); - timeout = !service_wait_exit( - SERVICE_MAINTENANCE - , 3 * USEC_PER_SEC); + service_wait_exit(SERVICE_COLLECTORS | SERVICE_STREAMING, 3 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS); - delta_shutdown_time("stop replication, exporters, health and web servers threads"); - - timeout = !service_wait_exit( - SERVICE_EXPORTERS - | SERVICE_HEALTH - | SERVICE_WEB_SERVER - | SERVICE_HTTPD - , 3 * USEC_PER_SEC); - - delta_shutdown_time("stop collectors and streaming threads"); - - timeout = !service_wait_exit( - SERVICE_COLLECTORS - | SERVICE_STREAMING - , 3 * USEC_PER_SEC); - - delta_shutdown_time("stop replication threads"); - - timeout = !service_wait_exit( - SERVICE_REPLICATION // replication has to be stopped after STREAMING, because it cleans up ARAL - , 3 * USEC_PER_SEC); - - delta_shutdown_time("prepare metasync shutdown"); + service_wait_exit(SERVICE_REPLICATION, 3 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_REPLICATION_THREADS); metadata_sync_shutdown_prepare(); - - delta_shutdown_time("disable ML detection and training threads"); + watcher_step_complete(WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN); ml_stop_threads(); ml_fini(); + watcher_step_complete(WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS); - delta_shutdown_time("stop context thread"); - - timeout = !service_wait_exit( - SERVICE_CONTEXT - , 3 * USEC_PER_SEC); - - delta_shutdown_time("clear web client cache"); + service_wait_exit(SERVICE_CONTEXT, 3 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_CONTEXT_THREAD); web_client_cache_destroy(); + watcher_step_complete(WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE); - delta_shutdown_time("stop aclk threads"); - - timeout = !service_wait_exit( - SERVICE_ACLK - , 3 * USEC_PER_SEC); - - delta_shutdown_time("stop all remaining worker threads"); + service_wait_exit(SERVICE_ACLK, 3 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_ACLK_THREADS); - timeout = !service_wait_exit(~0, 10 * USEC_PER_SEC); - - delta_shutdown_time("cancel main threads"); + service_wait_exit(~0, 10 * USEC_PER_SEC); + watcher_step_complete(WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS); cancel_main_threads(); + watcher_step_complete(WATCHER_STEP_ID_CANCEL_MAIN_THREADS); + + if (ret) + { + watcher_step_complete(WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS); + watcher_step_complete(WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS); + watcher_step_complete(WATCHER_STEP_ID_STOP_METASYNC_THREADS); - if(!ret) { + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH); + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING); + watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS); + } + else + { // exit cleanly #ifdef ENABLE_DBENGINE if(dbengine_enabled) { - delta_shutdown_time("flush dbengine tiers"); for (size_t tier = 0; tier < storage_tiers; tier++) rrdeng_prepare_exit(multidb_ctx[tier]); @@ -439,21 +399,16 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re } } #endif + watcher_step_complete(WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS); - // free the database - delta_shutdown_time("stop collection for all hosts"); - - // rrdhost_free_all(); rrd_finalize_collection_for_all_hosts(); - - delta_shutdown_time("stop metasync threads"); + watcher_step_complete(WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS); metadata_sync_shutdown(); + watcher_step_complete(WATCHER_STEP_ID_STOP_METASYNC_THREADS); #ifdef ENABLE_DBENGINE if(dbengine_enabled) { - delta_shutdown_time("wait for dbengine collectors to finish"); - size_t running = 1; size_t count = 10; while(running && count) { @@ -467,52 +422,55 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re } count--; } - - delta_shutdown_time("wait for dbengine main cache to finish flushing"); + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH); while (pgc_hot_and_dirty_entries(main_cache)) { pgc_flush_all_hot_and_dirty_pages(main_cache, PGC_SECTION_ALL); sleep_usec(100 * USEC_PER_MS); } + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING); - delta_shutdown_time("stop dbengine tiers"); for (size_t tier = 0; tier < storage_tiers; tier++) rrdeng_exit(multidb_ctx[tier]); - rrdeng_enq_cmd(NULL, RRDENG_OPCODE_SHUTDOWN_EVLOOP, NULL, NULL, STORAGE_PRIORITY_BEST_EFFORT, NULL, NULL); + watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS); + } else { + // Skip these steps + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH); + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING); + watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS); } +#else + // Skip these steps + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH); + watcher_step_complete(WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING); + watcher_step_complete(WATCHER_STEP_ID_STOP_DBENGINE_TIERS); #endif } - delta_shutdown_time("close SQL context db"); - sql_close_context_database(); - - delta_shutdown_time("closed SQL main db"); + watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB); sql_close_database(); + watcher_step_complete(WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB); // unlink the pid if(pidfile[0]) { - delta_shutdown_time("remove pid file"); - if(unlink(pidfile) != 0) netdata_log_error("EXIT: cannot unlink pidfile '%s'.", pidfile); } + watcher_step_complete(WATCHER_STEP_ID_REMOVE_PID_FILE); #ifdef ENABLE_HTTPS - delta_shutdown_time("free openssl structures"); netdata_ssl_cleanup(); #endif - - delta_shutdown_time("remove incomplete shutdown file"); + watcher_step_complete(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES); (void) unlink(agent_incomplete_shutdown_file); - - delta_shutdown_time("exit"); - - usec_t ended_ut = now_monotonic_usec(); - netdata_log_info("NETDATA SHUTDOWN: completed in %llu ms - netdata is now exiting - bye bye...", (ended_ut - started_ut) / USEC_PER_MS); + watcher_step_complete(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE); + + watcher_shutdown_end(); + watcher_thread_stop(); #ifdef ENABLE_SENTRY if (ret) @@ -1917,6 +1875,9 @@ int main(int argc, char **argv) { load_cloud_conf(0); } + // @stelfrag: Where is the right place to call this? + watcher_thread_start(); + // ------------------------------------------------------------------------ // initialize netdata { diff --git a/src/daemon/static_threads.c b/src/daemon/static_threads.c index 781c22b031..e03819761e 100644 --- a/src/daemon/static_threads.c +++ b/src/daemon/static_threads.c @@ -14,7 +14,7 @@ void *service_main(void *ptr); void *statsd_main(void *ptr); void *timex_main(void *ptr); void *profile_main(void *ptr); -void *replication_thread_main(void *ptr __maybe_unused); +void *replication_thread_main(void *ptr); extern bool global_statistics_enabled; diff --git a/src/daemon/watcher.c b/src/daemon/watcher.c new file mode 100644 index 0000000000..2a3bcf544b --- /dev/null +++ b/src/daemon/watcher.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "daemon/common.h" +#include "watcher.h" + +watcher_step_t *watcher_steps; + +static struct completion shutdown_begin_completion; +static struct completion shutdown_end_completion; +static netdata_thread_t watcher_thread; + +void watcher_shutdown_begin(void) { + completion_mark_complete(&shutdown_begin_completion); +} + +void watcher_shutdown_end(void) { + completion_mark_complete(&shutdown_end_completion); +} + +void watcher_step_complete(watcher_step_id_t step_id) { + completion_mark_complete(&watcher_steps[step_id].p); +} + +void *watcher_main(void *arg) +{ + UNUSED(arg); + + netdata_log_debug(D_SYSTEM, "Watcher thread started"); + + completion_wait_for(&shutdown_begin_completion); + usec_t shutdown_start_time = now_monotonic_usec(); + + // TODO: + // - add a version of completion_wait_for with timeout + // - check the step's duration and abort when the timeout has expired. + for (int step_id = 0; step_id != WATCHER_STEP_ID_MAX; step_id++) { + usec_t step_start_time = now_monotonic_usec(); + completion_wait_for(&watcher_steps[step_id].p); + usec_t step_end_time = now_monotonic_usec(); + + usec_t step_duration = step_end_time - step_start_time; + netdata_log_info("shutdown step: [%d/%d] - '%s' finished in %llu milliseconds", + step_id + 1, + WATCHER_STEP_ID_MAX, + watcher_steps[step_id].msg, + step_duration / USEC_PER_MS); + } + + netdata_log_error("Shutdown process started"); + + completion_wait_for(&shutdown_end_completion); + usec_t shutdown_end_time = now_monotonic_usec(); + + usec_t shutdown_duration = shutdown_end_time - shutdown_start_time; + netdata_log_error("Shutdown process ended in %llu milliseconds", + shutdown_duration / USEC_PER_MS); + + return NULL; +} + +void watcher_thread_start() { + watcher_steps = callocz(WATCHER_STEP_ID_MAX, sizeof(watcher_step_t)); + + watcher_steps[WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE].msg = + "create shutdown file"; + watcher_steps[WATCHER_STEP_ID_DBENGINE_EXIT_MODE].msg = + "dbengine exit mode"; + watcher_steps[WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS].msg = + "close webrtc connections"; + watcher_steps[WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK].msg = + "disable maintenance, new queries, new web requests, new streaming connections and aclk"; + watcher_steps[WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD].msg = + "stop maintenance thread"; + watcher_steps[WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS].msg = + "stop exporters, health and web servers threads"; + watcher_steps[WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS].msg = + "stop collectors and streaming threads"; + watcher_steps[WATCHER_STEP_ID_STOP_REPLICATION_THREADS].msg = + "stop replication threads"; + watcher_steps[WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN].msg = + "prepare metasync shutdown"; + watcher_steps[WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS].msg = + "disable ML detection and training threads"; + watcher_steps[WATCHER_STEP_ID_STOP_CONTEXT_THREAD].msg = + "stop context thread"; + watcher_steps[WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE].msg = + "clear web client cache"; + watcher_steps[WATCHER_STEP_ID_STOP_ACLK_THREADS].msg = + "stop aclk threads"; + watcher_steps[WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS].msg = + "stop all remaining worker threads"; + watcher_steps[WATCHER_STEP_ID_CANCEL_MAIN_THREADS].msg = + "cancel main threads"; + watcher_steps[WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS].msg = + "flush dbengine tiers"; + watcher_steps[WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS].msg = + "stop collection for all hosts"; + watcher_steps[WATCHER_STEP_ID_STOP_METASYNC_THREADS].msg = + "stop metasync threads"; + watcher_steps[WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH].msg = + "wait for dbengine collectors to finish"; + watcher_steps[WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING].msg = + "wait for dbengine main cache to finish flushing"; + watcher_steps[WATCHER_STEP_ID_STOP_DBENGINE_TIERS].msg = + "stop dbengine tiers"; + watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB].msg = + "close SQL context db"; + watcher_steps[WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB].msg = + "close SQL main db"; + watcher_steps[WATCHER_STEP_ID_REMOVE_PID_FILE].msg = + "remove pid file"; + watcher_steps[WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES].msg = + "free openssl structures"; + watcher_steps[WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE].msg = + "remove incomplete shutdown file"; + + for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) { + completion_init(&watcher_steps[i].p); + } + + completion_init(&shutdown_begin_completion); + completion_init(&shutdown_end_completion); + + netdata_thread_create(&watcher_thread, "P[WATCHER]", NETDATA_THREAD_OPTION_JOINABLE, watcher_main, NULL); +} + +void watcher_thread_stop() { + netdata_thread_join(watcher_thread, NULL); + + for (size_t i = 0; i != WATCHER_STEP_ID_MAX; i++) { + completion_destroy(&watcher_steps[i].p); + } + + completion_destroy(&shutdown_begin_completion); + completion_destroy(&shutdown_end_completion); + + freez(watcher_steps); +} diff --git a/src/daemon/watcher.h b/src/daemon/watcher.h new file mode 100644 index 0000000000..6c9ce15523 --- /dev/null +++ b/src/daemon/watcher.h @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef DAEMON_WATCHER_H +#define DAEMON_WATCHER_H + +#include "libnetdata/libnetdata.h" + +typedef enum { + WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE = 0, + WATCHER_STEP_ID_DBENGINE_EXIT_MODE, + WATCHER_STEP_ID_CLOSE_WEBRTC_CONNECTIONS, + WATCHER_STEP_ID_DISABLE_MAINTENANCE_NEW_QUERIES_NEW_WEB_REQUESTS_NEW_STREAMING_CONNECTIONS_AND_ACLK, + WATCHER_STEP_ID_STOP_MAINTENANCE_THREAD, + WATCHER_STEP_ID_STOP_EXPORTERS_HEALTH_AND_WEB_SERVERS_THREADS, + WATCHER_STEP_ID_STOP_COLLECTORS_AND_STREAMING_THREADS, + WATCHER_STEP_ID_STOP_REPLICATION_THREADS, + WATCHER_STEP_ID_PREPARE_METASYNC_SHUTDOWN, + WATCHER_STEP_ID_DISABLE_ML_DETECTION_AND_TRAINING_THREADS, + WATCHER_STEP_ID_STOP_CONTEXT_THREAD, + WATCHER_STEP_ID_CLEAR_WEB_CLIENT_CACHE, + WATCHER_STEP_ID_STOP_ACLK_THREADS, + WATCHER_STEP_ID_STOP_ALL_REMAINING_WORKER_THREADS, + WATCHER_STEP_ID_CANCEL_MAIN_THREADS, + WATCHER_STEP_ID_FLUSH_DBENGINE_TIERS, + WATCHER_STEP_ID_STOP_COLLECTION_FOR_ALL_HOSTS, + WATCHER_STEP_ID_STOP_METASYNC_THREADS, + WATCHER_STEP_ID_WAIT_FOR_DBENGINE_COLLECTORS_TO_FINISH, + WATCHER_STEP_ID_WAIT_FOR_DBENGINE_MAIN_CACHE_TO_FINISH_FLUSHING, + WATCHER_STEP_ID_STOP_DBENGINE_TIERS, + WATCHER_STEP_ID_CLOSE_SQL_CONTEXT_DB, + WATCHER_STEP_ID_CLOSE_SQL_MAIN_DB, + WATCHER_STEP_ID_REMOVE_PID_FILE, + WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES, + WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE, + + // Always keep this as the last enum value + WATCHER_STEP_ID_MAX +} watcher_step_id_t; + +typedef struct { + const char *msg; + struct completion p; +} watcher_step_t; + +extern watcher_step_t *watcher_steps; + +void watcher_thread_start(void); +void watcher_thread_stop(void); + +void watcher_shutdown_begin(void); +void watcher_shutdown_end(void); + +void watcher_step_complete(watcher_step_id_t step_id); + +#endif /* DAEMON_WATCHER_H */ |