Merge pull request #3184 from ktsaou/master

fix deadlock on exit of static netdata build

37 files changed, 1117 insertions, 868 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f0c0d519b1..b4c50e55b1 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -63,7 +63,8 @@ set(NETDATA_LINUX_FILES
         src/proc_uptime.c
         src/sys_kernel_mm_ksm.c
         src/sys_fs_cgroup.c
-        src/sys_fs_btrfs.c)
+        src/sys_fs_btrfs.c
+        src/threads.c src/threads.h)
 
 set(NETDATA_COMMON_FILES
         src/adaptive_resortable_list.c
@@ -94,6 +95,7 @@ set(NETDATA_COMMON_FILES
         src/health_json.c
         src/health_log.c
         src/inlined.h
+        src/locks.c
         src/locks.h
         src/log.c
         src/log.h
@@ -189,6 +191,8 @@ set(APPS_PLUGIN_SOURCE_FILES
         src/clocks.c
         src/clocks.h
         src/inlined.h
+        src/locks.c
+        src/locks.h
         src/log.c
         src/log.h
         src/procfile.c
@@ -205,6 +209,8 @@ set(FREEIPMI_PLUGIN_SOURCE_FILES
         src/clocks.c
         src/clocks.h
         src/inlined.h
+        src/locks.c
+        src/locks.h
         src/log.c
         src/log.h
         src/procfile.c
@@ -219,6 +225,8 @@ set(CGROUP_NETWORK_SOURCE_FILES
         src/clocks.c
         src/clocks.h
         src/inlined.h
+        src/locks.c
+        src/locks.h
         src/log.c
         src/log.h
         src/procfile.c
diff --git a/makeself/build-x86_64-static.sh b/makeself/build-x86_64-static.sh
index 8c84039f3e..3576660935 100755
--- a/makeself/build-x86_64-static.sh
+++ b/makeself/build-x86_64-static.sh
@@ -4,7 +4,7 @@
 
 set -e
 
-DOCKER_CONTAINER_NAME="netdata-package-x86_64-static"
+DOCKER_CONTAINER_NAME="netdata-package-x86_64-static-alpine37"
 
 if ! sudo docker inspect "${DOCKER_CONTAINER_NAME}" >/dev/null 2>&1
 then
@@ -21,7 +21,7 @@ then
     # inside the container and runs the script install-alpine-packages.sh
     # (also inside the container)
     #
-    run sudo docker run -v $(pwd):/usr/src/netdata.git:rw alpine:3.6 \
+    run sudo docker run -v $(pwd):/usr/src/netdata.git:rw alpine:3.7 \
         /bin/sh /usr/src/netdata.git/makeself/install-alpine-packages.sh
 
     # save the changes made permanently
diff --git a/makeself/jobs/70-netdata-git.install.sh b/makeself/jobs/70-netdata-git.install.sh
index b85481492c..fea3a88bd6 100755
--- a/makeself/jobs/70-netdata-git.install.sh
+++ b/makeself/jobs/70-netdata-git.install.sh
@@ -9,6 +9,7 @@ then
     export CFLAGS="-static -O3"
 else
     export CFLAGS="-static -O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -D_FORTIFY_SOURCE=2 -DNETDATA_INTERNAL_CHECKS=1"
+#    export CFLAGS="-static -O1 -ggdb -Wall -Wextra -Wformat-signedness"
 fi
 
 if [ ! -z "${NETDATA_INSTALL_PATH}" -a -d "${NETDATA_INSTALL_PATH}/etc" ]
diff --git a/src/Makefile.am b/src/Makefile.am
index 577871392f..cbb8b0ccbc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -70,6 +70,7 @@ netdata_SOURCES = \
 	health_json.c \
 	health_log.c \
 	inlined.h \
+	locks.c \
 	locks.h \
 	log.c \
 	log.h \
@@ -127,6 +128,8 @@ netdata_SOURCES = \
 	statsd.h \
 	storage_number.c \
 	storage_number.h \
+	threads.c \
+	threads.h \
 	unit_test.c \
 	unit_test.h \
 	url.c \
@@ -223,13 +226,22 @@ netdata_LDADD = \
 
 apps_plugin_SOURCES = \
 	apps_plugin.c \
-	avl.c avl.h \
-	clocks.c clocks.h \
-	common.c common.h \
+	avl.c \
+	avl.h \
+	clocks.c \
+	clocks.h \
+	common.c \
+	common.h \
 	inlined.h \
+	locks.c \
+	locks.h \
 	log.c log.h \
-	procfile.c procfile.h \
-	web_buffer.c web_buffer.h \
+	procfile.c \
+	procfile.h \
+	threads.c \
+	threads.h \
+	web_buffer.c \
+	web_buffer.h \
 	$(NULL)
 
 if FREEBSD
@@ -245,11 +257,18 @@ apps_plugin_LDADD = \
 
 freeipmi_plugin_SOURCES = \
 	freeipmi_plugin.c \
-	clocks.c clocks.h \
-	common.c common.h \
+	clocks.c \
+	clocks.h \
+	common.c \
+	common.h \
 	inlined.h \
+	locks.c \
+	locks.h \
 	log.c log.h \
-	procfile.c procfile.h \
+	procfile.c \
+	procfile.h \
+	threads.c \
+	threads.h \
 	$(NULL)
 
 freeipmi_plugin_LDADD = \
@@ -258,13 +277,23 @@ freeipmi_plugin_LDADD = \
 
 cgroup_network_SOURCES = \
     cgroup-network.c \
-	clocks.c clocks.h \
-    common.c common.h \
+	clocks.c \
+	clocks.h \
+    common.c \
+    common.h \
     inlined.h \
-    log.c log.h \
-	procfile.c procfile.h \
-	popen.c popen.h \
-	signals.c signals.h \
+	locks.c \
+	locks.h \
+    log.c \
+    log.h \
+	procfile.c \
+	procfile.h \
+	popen.c \
+	popen.h \
+	signals.c \
+	signals.h \
+	threads.c \
+	threads.h \
     $(NULL)
 
 cgroup_network_LDADD = \
diff --git a/src/backends.c b/src/backends.c
index 6db8be94a0..6160450ff3 100644
--- a/src/backends.c
+++ b/src/backends.c
@@ -498,23 +498,24 @@ inline uint32_t backend_parse_data_source(const char *source, uint32_t mode) {
     return mode;
 }
 
+static void backends_main_cleanup(void *ptr) {
+    struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+    if(static_thread->enabled) {
+        static_thread->enabled = 0;
+
+        info("%s: cleaning up...", netdata_thread_tag());
+    }
+}
+
 void *backends_main(void *ptr) {
+    netdata_thread_cleanup_push(backends_main_cleanup, ptr);
+
     int default_port = 0;
     int sock = -1;
-    struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
-
     BUFFER *b = buffer_create(1), *response = buffer_create(1);
     int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, uint32_t) = NULL;
     int (*backend_response_checker)(BUFFER *) = NULL;
 
-    info("BACKEND: thread created with task id %d", gettid());
-
-    if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
-        error("BACKEND: cannot set pthread cancel type to DEFERRED.");
-
-    if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
-        error("BACKEND: cannot set pthread cancel state to ENABLE.");
-
     // ------------------------------------------------------------------------
     // collect configuration options
 
@@ -664,7 +665,7 @@ void *backends_main(void *ptr) {
     heartbeat_t hb;
     heartbeat_init(&hb);
 
-    for(;;) {
+    while(!netdata_exit) {
 
         // ------------------------------------------------------------------------
         // Wait for the next iteration point.
@@ -676,10 +677,7 @@ void *backends_main(void *ptr) {
         // ------------------------------------------------------------------------
         // add to the buffer the data we need to send to the backend
 
-        int pthreadoldcancelstate;
-
-        if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &pthreadoldcancelstate) != 0))
-            error("BACKEND: cannot set pthread cancel state to DISABLE.");
+        netdata_thread_disable_cancelability();
 
         size_t count_hosts = 0;
         size_t count_charts_total = 0;
@@ -728,10 +726,9 @@ void *backends_main(void *ptr) {
         }
         rrd_unlock();
 
-        debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts);
+        netdata_thread_enable_cancelability();
 
-        if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0))
-            error("BACKEND: cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate);
+        debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts);
 
         // ------------------------------------------------------------------------
 
@@ -918,9 +915,6 @@ cleanup:
     buffer_free(b);
     buffer_free(response);
 
-    info("BACKEND: thread exiting");
-
-    static_thread->enabled = 0;
-    pthread_exit(NULL);
+    netdata_thread_cleanup_pop(1);
     return NULL;
 }
diff --git a/src/common.c b/src/common.c
index a976e96eb3..1c53e7ae00 100644
--- a/src/common.c
+++ b/src/common.c
@@ -1116,22 +1116,6 @@ int fd_is_valid(int fd) {
     return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
 }
 
-pid_t gettid(void) {
-#ifdef __FreeBSD__
-    return (pid_t)pthread_getthreadid_np();
-#elif defined(__APPLE__)
-#if (defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1060)
-    uint64_t curthreadid;
-    pthread_threadid_np(NULL, &curthreadid);
-    return (pid_t)curthreadid;
-#else /* __MAC_OS_X_VERSION_MIN_REQUIRED */
-    return (pid_t)pthread_self;
-#endif /* __MAC_OS_X_VERSION_MIN_REQUIRED */
-#else /* __APPLE__*/
-    return (pid_t)syscall(SYS_gettid);
-#endif /* __FreeBSD__, __APPLE__*/
-}
-
 char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len) {
     char *s = fgets(buf, (int)buf_size, fp);
     if (!s) return NULL;
diff --git a/src/common.h b/src/common.h
index 688d5fed9a..951a1badf1 100644
--- a/src/common.h
+++ b/src/common.h
@@ -172,6 +172,7 @@
 
 #include "clocks.h"
 #include "log.h"
+#include "threads.h"
 #include "locks.h"
 #include "simple_pattern.h"
 #include "avl.h"
@@ -294,8 +295,6 @@ extern int fd_is_valid(int fd);
 
 extern int enable_ksm;
 
-extern pid_t gettid(void);
-
 extern int sleep_usec(usec_t usec);
 
 extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);
diff --git a/src/health.c b/src/health.c
index dfa7007b96..cd0ad43fae 100644
--- a/src/health.c
+++ b/src/health.c
@@ -338,22 +338,22 @@ static inline int check_if_resumed_from_suspention(void) {
     return ret;
 }
 
-void *health_main(void *ptr) {
+static void health_main_cleanup(void *ptr) {
     struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+    if(static_thread->enabled) {
+        static_thread->enabled = 0;
 
-    info("HEALTH thread created with task id %d", gettid());
-
-    if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
-        error("Cannot set pthread cancel type to DEFERRED.");
+        info("%s: cleaning up...", netdata_thread_tag());
+    }
+}
 
-    if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
-        error("Cannot set pthread cancel state to ENABLE.");
+void *health_main(void *ptr) {
+    BUFFER *wb = buffer_create(100);
+    netdata_thread_cleanup_push(health_main_cleanup, ptr);
 
     int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
     if(min_run_every < 1) min_run_every = 1;
 
-    BUFFER *wb = buffer_create(100);
-
     time_t now                = now_realtime_sec();
     time_t hibernation_delay  = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60);
 
@@ -362,7 +362,7 @@ void *health_main(void *ptr) {
         loop++;
         debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
 
-        int oldstate, runnable = 0, apply_hibernation_delay = 0;
+        int runnable = 0, apply_hibernation_delay = 0;
         time_t next_run = now + min_run_every;
         RRDCALC *rc;
 
@@ -374,9 +374,6 @@ void *health_main(void *ptr) {
             );
         }
 
-        if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
-            error("Cannot set pthread cancel state to DISABLE.");
-
         rrd_rdlock();
 
         RRDHOST *host;
@@ -721,9 +718,6 @@ void *health_main(void *ptr) {
 
         rrd_unlock();
 
-        if(unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
-            error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
-
         if(unlikely(netdata_exit))
             break;
 
@@ -740,9 +734,6 @@ void *health_main(void *ptr) {
 
     buffer_free(wb);
 
-    info("HEALTH thread exiting");
-
-    static_thread->enabled = 0;
-    pthread_exit(NULL);
+    netdata_thread_cleanup_pop(1);
     return NULL;
 }
diff --git a/src/locks.c b/src/locks.c
new file mode 100644
index 0000000000..9ab9abcfe6
--- /dev/null
+++ b/src/locks.c
@@ -0,0 +1,319 @@
+#include "common.h"
+
+// ----------------------------------------------------------------------------
+// automatic thread cancelability management, based on locks
+
+static __thread int netdata_thread_first_cancelability = 0;
+static __thread int netdata_thread_lock_cancelability = 0;
+
+inline void netdata_thread_disable_cancelability(void) {
+    int old;
+    int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);
+    if(ret != 0)
+        error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret);
+    else {
+        if(!netdata_thread_lock_cancelability)
+            netdata_thread_first_cancelability = old;
+
+        netdata_thread_lock_cancelability++;
+    }
+}
+
+inline void netdata_thread_enable_cancelability(void) {
+    if(netdata_thread_lock_cancelability < 1) {
+        error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d on thread %s - results will be undefined - please report this!", netdata_thread_lock_cancelability, netdata_thread_tag());
+    }
+    else if(netdata_thread_lock_cancelability == 1) {
+        int old = 1;
+        int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old);
+        if(ret != 0)
+            error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret);
+        else {
+            if(!old)
+                error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability on thread %s was changed, expected ENABLED, found DISABLED - please report this!", netdata_thread_tag());
+        }
+
+        netdata_thread_lock_cancelability = 0;
+    }
+    else
+        netdata_thread_lock_cancelability--;
+}
+
+// ----------------------------------------------------------------------------
+// mutex
+
+int __netdata_mutex_init(netdata_mutex_t *mutex) {
+    int ret = pthread_mutex_init(mutex, NULL);
+    if(unlikely(ret != 0))
+        error("MUTEX_LOCK: failed to initialize (code %d).", ret);
+    return ret;
+}
+
+int __netdata_mutex_lock(netdata_mutex_t *mutex) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_mutex_lock(mutex);
+    if(unlikely(ret != 0)) {
+        netdata_thread_enable_cancelability();
+        error("MUTEX_LOCK: failed to get lock (code %d)", ret);
+    }
+    return ret;
+}
+
+int __netdata_mutex_trylock(netdata_mutex_t *mutex) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_mutex_trylock(mutex);
+    if(ret != 0)
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int __netdata_mutex_unlock(netdata_mutex_t *mutex) {
+    int ret = pthread_mutex_unlock(mutex);
+    if(unlikely(ret != 0))
+        error("MUTEX_LOCK: failed to unlock (code %d).", ret);
+    else
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int netdata_mutex_init_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_init(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_mutex_lock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_lock(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_lock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_mutex_trylock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_trylock(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_trylock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_mutex_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_mutex_t *mutex) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) from %lu@%s, %s()", mutex, line, file, function);
+    }
+
+    int ret = __netdata_mutex_unlock(mutex);
+
+    debug(D_LOCKS, "MUTEX_LOCK: netdata_mutex_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", mutex, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+
+// ----------------------------------------------------------------------------
+// r/w lock
+
+int __netdata_rwlock_destroy(netdata_rwlock_t *rwlock) {
+    int ret = pthread_rwlock_destroy(rwlock);
+    if(unlikely(ret != 0))
+        error("RW_LOCK: failed to destroy lock (code %d)", ret);
+    return ret;
+}
+
+int __netdata_rwlock_init(netdata_rwlock_t *rwlock) {
+    int ret = pthread_rwlock_init(rwlock, NULL);
+    if(unlikely(ret != 0))
+        error("RW_LOCK: failed to initialize lock (code %d)", ret);
+    return ret;
+}
+
+int __netdata_rwlock_rdlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_rdlock(rwlock);
+    if(unlikely(ret != 0)) {
+        netdata_thread_enable_cancelability();
+        error("RW_LOCK: failed to obtain read lock (code %d)", ret);
+    }
+
+    return ret;
+}
+
+int __netdata_rwlock_wrlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_wrlock(rwlock);
+    if(unlikely(ret != 0)) {
+        error("RW_LOCK: failed to obtain write lock (code %d)", ret);
+        netdata_thread_enable_cancelability();
+    }
+
+    return ret;
+}
+
+int __netdata_rwlock_unlock(netdata_rwlock_t *rwlock) {
+    int ret = pthread_rwlock_unlock(rwlock);
+    if(unlikely(ret != 0))
+        error("RW_LOCK: failed to release lock (code %d)", ret);
+    else
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int __netdata_rwlock_tryrdlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_tryrdlock(rwlock);
+    if(ret != 0)
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {
+    netdata_thread_disable_cancelability();
+
+    int ret = pthread_rwlock_trywrlock(rwlock);
+    if(ret != 0)
+        netdata_thread_enable_cancelability();
+
+    return ret;
+}
+
+
+int netdata_rwlock_destroy_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_destroy(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_destroy(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_init_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_init(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_init(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_rdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_rdlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_rdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_wrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_wrlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_wrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_unlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_unlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_unlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_tryrdlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_tryrdlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_tryrdlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
+
+int netdata_rwlock_trywrlock_debug( const char *file, const char *function, const unsigned long line, netdata_rwlock_t *rwlock) {
+    usec_t start = 0;
+
+    if(unlikely(debug_flags & D_LOCKS)) {
+        start = now_boottime_usec();
+        debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) from %lu@%s, %s()", rwlock, line, file, function);
+    }
+
+    int ret = __netdata_rwlock_trywrlock(rwlock);
+
+    debug(D_LOCKS, "RW_LOCK: netdata_rwlock_trywrlock(0x%p) = %d in %llu usec, from %lu@%s, %s()", rwlock, ret, now_boottime_usec() - start, line, file, function);
+
+    return ret;
+}
diff --git a/src/locks.h b/src/locks.h
index 76533f636e..36962fef2f 100644
--- a/src/locks.h
+++ b/src/locks.h
@@ -1,276 +1,48 @@
 #ifndef NETDATA_LOCKS_H
 #define NETDATA_LOCKS_H
 
-// ----------------------------------------------------------------------------
-// mutex
-
 typedef pthread_mutex_t netdata_mutex_t;
-
 #define NETDATA_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
 
-static inline int __netdata_mutex_init(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_init(mutex, NULL);
-    if(unlikely(ret != 0))
-        error("MUTEX_LOCK: failed to initialize (code %d).", ret);
-    return ret;
-}
-
-static inline int __netdata_mutex_lock(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_lock(mutex);
-    if(unlikely(ret != 0))
-        error("MUTEX_LOCK: failed to get lock (code %d)", ret);
-    return ret;
-}
-
-static inline int __netdata_mutex_trylock(netdata_mutex_t *mutex) {
-    int ret = pthread_mutex_trylock(mutex);
-    return ret;
-}
+typedef pthread_rwlock_t netdata_rwlock_t;
+#define NETDATA_RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER
 
-static inline int __netdata_mutex_unlock(netdata_mutex_t *mut