modularized all source code (#4391)

* modularized all external plugins

* added README.md in plugins

* fixed title

* fixed typo

* relative link to external plugins

* external plugins configuration README

* added plugins link

* remove plugins link

* plugin names are links

* added links to external plugins

* removed unecessary spacing

* list to table

* added language

* fixed typo

* list to table on internal plugins

* added more documentation to internal plugins

* moved python, node, and bash code and configs into the external plugins

* added statsd README

* fix bug with corrupting config.h every 2nd compilation

* moved all config files together with their code

* more documentation

* diskspace info

* fixed broken links in apps.plugin

* added backends docs

* updated plugins readme

* move nc-backend.sh to backends

* created daemon directory

* moved all code outside src/

* fixed readme identation

* renamed plugins.d.plugin to plugins.d

* updated readme

* removed linux- from linux plugins

* updated readme

* updated readme

* updated readme

* updated readme

* updated readme

* updated readme

* fixed README.md links

* fixed netdata tree links

* updated codacy, codeclimate and lgtm excluded paths

* update CMakeLists.txt

* updated automake options at top directory

* libnetdata slit into directories

* updated READMEs

* updated READMEs

* updated ARL docs

* updated ARL docs

* moved /plugins to /collectors

* moved all external plugins outside plugins.d

* updated codacy, codeclimate, lgtm

* updated README

* updated url

* updated readme

* updated readme

* updated readme

* updated readme

* moved api and web into webserver

* web/api web/gui web/server

* modularized webserver

* removed web/gui/version.txt

15 files changed, 2143 insertions, 0 deletions

diff --git a/registry/Makefile.am b/registry/Makefile.am
new file mode 100644
index 0000000000..1cb69ed99a
--- /dev/null
+++ b/registry/Makefile.am
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+
+dist_noinst_DATA = \
+	README.md \
+	$(NULL)
diff --git a/registry/README.md b/registry/README.md
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/registry/README.md
diff --git a/registry/registry.c b/registry/registry.c
new file mode 100644
index 0000000000..4f97eb58fd
--- /dev/null
+++ b/registry/registry.c
@@ -0,0 +1,415 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../daemon/common.h"
+#include "registry_internals.h"
+
+#define REGISTRY_STATUS_OK "ok"
+#define REGISTRY_STATUS_FAILED "failed"
+#define REGISTRY_STATUS_DISABLED "disabled"
+
+// ----------------------------------------------------------------------------
+// REGISTRY concurrency locking
+
+static inline void registry_lock(void) {
+    netdata_mutex_lock(&registry.lock);
+}
+
+static inline void registry_unlock(void) {
+    netdata_mutex_unlock(&registry.lock);
+}
+
+
+// ----------------------------------------------------------------------------
+// COOKIES
+
+static void registry_set_cookie(struct web_client *w, const char *guid) {
+    char edate[100];
+    time_t et = now_realtime_sec() + registry.persons_expiration;
+    struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);
+    strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);
+
+    snprintfz(w->cookie1, NETDATA_WEB_REQUEST_COOKIE_SIZE, NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s", guid, edate);
+
+    if(registry.registry_domain && registry.registry_domain[0])
+        snprintfz(w->cookie2, NETDATA_WEB_REQUEST_COOKIE_SIZE, NETDATA_REGISTRY_COOKIE_NAME "=%s; Domain=%s; Expires=%s", guid, registry.registry_domain, edate);
+}
+
+static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PERSON *p) {
+    registry_set_cookie(w, p->guid);
+}
+
+
+// ----------------------------------------------------------------------------
+// JSON GENERATION
+
+static inline void registry_json_header(RRDHOST *host, struct web_client *w, const char *action, const char *status) {
+    buffer_flush(w->response.data);
+    w->response.data->contenttype = CT_APPLICATION_JSON;
+    buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",
+            action, status, host->registry_hostname, host->machine_guid);
+}
+
+static inline void registry_json_footer(struct web_client *w) {
+    buffer_strcat(w->response.data, "\n}\n");
+}
+
+static inline int registry_json_disabled(RRDHOST *host, struct web_client *w, const char *action) {
+    registry_json_header(host, w, action, REGISTRY_STATUS_DISABLED);
+
+    buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
+            registry.registry_to_announce);
+
+    registry_json_footer(w);
+    return 200;
+}
+
+
+// ----------------------------------------------------------------------------
+// CALLBACKS FOR WALKING THROUGH REGISTRY OBJECTS
+
+// structure used be the callbacks below
+struct registry_json_walk_person_urls_callback {
+    REGISTRY_PERSON *p;
+    REGISTRY_MACHINE *m;
+    struct web_client *w;
+    int count;
+};
+
+// callback for rendering PERSON_URLs
+static int registry_json_person_url_callback(void *entry, void *data) {
+    REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry;
+    struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
+    struct web_client *w = c->w;
+
+    if(unlikely(c->count++))
+        buffer_strcat(w->response.data, ",");
+
+    buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]",
+            pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->machine_name);
+
+    return 0;
+}
+
+// callback for rendering MACHINE_URLs
+static int registry_json_machine_url_callback(void *entry, void *data) {
+    REGISTRY_MACHINE_URL *mu = (REGISTRY_MACHINE_URL *)entry;
+    struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
+    struct web_client *w = c->w;
+    REGISTRY_MACHINE *m = c->m;
+
+    if(unlikely(c->count++))
+        buffer_strcat(w->response.data, ",");
+
+    buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]",
+            m->guid, mu->url->url, mu->last_t, mu->usages);
+
+    return 1;
+}
+
+// ----------------------------------------------------------------------------
+
+// structure used be the callbacks below
+struct registry_person_url_callback_verify_machine_exists_data {
+    REGISTRY_MACHINE *m;
+    int count;
+};
+
+static inline int registry_person_url_callback_verify_machine_exists(void *entry, void *data) {
+    struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data;
+    REGISTRY_PERSON_URL *pu = (REGISTRY_PERSON_URL *)entry;
+    REGISTRY_MACHINE *m = d->m;
+
+    if(pu->machine == m)
+        d->count++;
+
+    return 0;
+}
+
+// ----------------------------------------------------------------------------
+// public HELLO request
+
+int registry_request_hello_json(RRDHOST *host, struct web_client *w) {
+    registry_json_header(host, w, "hello", REGISTRY_STATUS_OK);
+
+    buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
+            registry.registry_to_announce);
+
+    registry_json_footer(w);
+    return 200;
+}
+
+// ----------------------------------------------------------------------------
+//public ACCESS request
+
+#define REGISTRY_VERIFY_COOKIES_GUID "give-me-back-this-cookie-now--please"
+
+// the main method for registering an access
+int registry_request_access_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
+    if(unlikely(!registry.enabled))
+        return registry_json_disabled(host, w, "access");
+
+    // ------------------------------------------------------------------------
+    // verify the browser supports cookies
+
+    if(registry.verify_cookies_redirects > 0 && !person_guid[0]) {
+        buffer_flush(w->response.data);
+        registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID);
+        w->response.data->contenttype = CT_APPLICATION_JSON;
+        buffer_sprintf(w->response.data, "{ \"status\": \"redirect\", \"registry\": \"%s\" }", registry.registry_to_announce);
+        return 200;
+    }
+
+    if(unlikely(person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID)))
+        person_guid[0] = '\0';
+
+    // ------------------------------------------------------------------------
+
+    registry_lock();
+
+    REGISTRY_PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when);
+    if(!p) {
+        registry_json_header(host, w, "access", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 412;
+    }
+
+    // set the cookie
+    registry_set_person_cookie(w, p);
+
+    // generate the response
+    registry_json_header(host, w, "access", REGISTRY_STATUS_OK);
+
+    buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\",\n\t\"urls\": [", p->guid);
+    struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 };
+    avl_traverse(&p->person_urls, registry_json_person_url_callback, &c);
+    buffer_strcat(w->response.data, "\n\t]\n");
+
+    registry_json_footer(w);
+    registry_unlock();
+    return 200;
+}
+
+// ----------------------------------------------------------------------------
+// public DELETE request
+
+// the main method for deleting a URL from a person
+int registry_request_delete_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
+    if(!registry.enabled)
+        return registry_json_disabled(host, w, "delete");
+
+    registry_lock();
+
+    REGISTRY_PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when);
+    if(!p) {
+        registry_json_header(host, w, "delete", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 412;
+    }
+
+    // generate the response
+    registry_json_header(host, w, "delete", REGISTRY_STATUS_OK);
+    registry_json_footer(w);
+    registry_unlock();
+    return 200;
+}
+
+// ----------------------------------------------------------------------------
+// public SEARCH request
+
+// the main method for searching the URLs of a netdata
+int registry_request_search_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
+    if(!registry.enabled)
+        return registry_json_disabled(host, w, "search");
+
+    registry_lock();
+
+    REGISTRY_MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when);
+    if(!m) {
+        registry_json_header(host, w, "search", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 404;
+    }
+
+    registry_json_header(host, w, "search", REGISTRY_STATUS_OK);
+
+    buffer_strcat(w->response.data, ",\n\t\"urls\": [");
+    struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };
+    dictionary_get_all(m->machine_urls, registry_json_machine_url_callback, &c);
+    buffer_strcat(w->response.data, "\n\t]\n");
+
+    registry_json_footer(w);
+    registry_unlock();
+    return 200;
+}
+
+// ----------------------------------------------------------------------------
+// SWITCH REQUEST
+
+// the main method for switching user identity
+int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when) {
+    if(!registry.enabled)
+        return registry_json_disabled(host, w, "switch");
+
+    (void)url;
+    (void)when;
+
+    registry_lock();
+
+    REGISTRY_PERSON *op = registry_person_find(person_guid);
+    if(!op) {
+        registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 430;
+    }
+
+    REGISTRY_PERSON *np = registry_person_find(new_person_guid);
+    if(!np) {
+        registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 431;
+    }
+
+    REGISTRY_MACHINE *m = registry_machine_find(machine_guid);
+    if(!m) {
+        registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 432;
+    }
+
+    struct registry_person_url_callback_verify_machine_exists_data data = { m, 0 };
+
+    // verify the old person has access to this machine
+    avl_traverse(&op->person_urls, registry_person_url_callback_verify_machine_exists, &data);
+    if(!data.count) {
+        registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 433;
+    }
+
+    // verify the new person has access to this machine
+    data.count = 0;
+    avl_traverse(&np->person_urls, registry_person_url_callback_verify_machine_exists, &data);
+    if(!data.count) {
+        registry_json_header(host, w, "switch", REGISTRY_STATUS_FAILED);
+        registry_json_footer(w);
+        registry_unlock();
+        return 434;
+    }
+
+    // set the cookie of the new person
+    // the user just switched identity
+    registry_set_person_cookie(w, np);
+
+    // generate the response
+    registry_json_header(host, w, "switch", REGISTRY_STATUS_OK);
+    buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\"", np->guid);
+    registry_json_footer(w);
+
+    registry_unlock();
+    return 200;
+}
+
+// ----------------------------------------------------------------------------
+// STATISTICS
+
+void registry_statistics(void) {
+    if(!registry.enabled) return;
+
+    static RRDSET *sts = NULL, *stc = NULL, *stm = NULL;
+
+    if(unlikely(!sts)) {
+        sts = rrdset_create_localhost(
+                "netdata"
+                , "registry_sessions"
+                , NULL
+                , "registry"
+                , NULL
+                , "NetData Registry Sessions"
+                , "session"
+                , "registry"
+                , "stats"
+                , 131000
+                , localhost->rrd_update_every
+                , RRDSET_TYPE_LINE
+        );
+
+        rrddim_add(sts, "sessions",  NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+    }
+    else rrdset_next(sts);
+
+    rrddim_set(sts, "sessions", registry.usages_count);
+    rrdset_done(sts);
+
+    // ------------------------------------------------------------------------
+
+    if(unlikely(!stc)) {
+        stc = rrdset_create_localhost(
+                "netdata"
+                , "registry_entries"
+                , NULL
+                , "registry"
+                , NULL
+                , "NetData Registry Entries"
+                , "entries"
+                , "registry"
+                , "stats"
+                , 131100
+                , localhost->rrd_update_every
+                , RRDSET_TYPE_LINE
+        );
+
+        rrddim_add(stc, "persons",        NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stc, "machines",       NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stc, "urls",           NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stc, "persons_urls",   NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stc, "machines_urls",  NULL,  1, 1, RRD_ALGORITHM_ABSOLUTE);
+    }
+    else rrdset_next(stc);
+
+    rrddim_set(stc, "persons",       registry.persons_count);
+    rrddim_set(stc, "machines",      registry.machines_count);
+    rrddim_set(stc, "urls",          registry.urls_count);
+    rrddim_set(stc, "persons_urls",  registry.persons_urls_count);
+    rrddim_set(stc, "machines_urls", registry.machines_urls_count);
+    rrdset_done(stc);
+
+    // ------------------------------------------------------------------------
+
+    if(unlikely(!stm)) {
+        stm = rrdset_create_localhost(
+                "netdata"
+                , "registry_mem"
+                , NULL
+                , "registry"
+                , NULL
+                , "NetData Registry Memory"
+                , "KB"
+                , "registry"
+                , "stats"
+                , 131300
+                , localhost->rrd_update_every
+                , RRDSET_TYPE_STACKED
+        );
+
+        rrddim_add(stm, "persons",        NULL,  1, 1024, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stm, "machines",       NULL,  1, 1024, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stm, "urls",           NULL,  1, 1024, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stm, "persons_urls",   NULL,  1, 1024, RRD_ALGORITHM_ABSOLUTE);
+        rrddim_add(stm, "machines_urls",  NULL,  1, 1024, RRD_ALGORITHM_ABSOLUTE);
+    }
+    else rrdset_next(stm);
+
+    rrddim_set(stm, "persons",       registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
+    rrddim_set(stm, "machines",      registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
+    rrddim_set(stm, "urls",          registry.urls_memory);
+    rrddim_set(stm, "persons_urls",  registry.persons_urls_memory);
+    rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE));
+    rrdset_done(stm);
+}
diff --git a/registry/registry.h b/registry/registry.h
new file mode 100644
index 0000000000..ab36de014f
--- /dev/null
+++ b/registry/registry.h
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+/*
+ * netdata registry
+ *
+ * this header file describes the public interface
+ * to the netdata registry
+ *
+ * only these high level functions are exposed
+ *
+ */
+
+// ----------------------------------------------------------------------------
+// TODO
+//
+// 1. the default tracking cookie expires in 1 year, but the persons are not
+//    removed from the db - this means the database only grows - ideally the
+//    database should be cleaned in registry_db_save() for both on-disk and
+//    on-memory entries.
+//
+//    Cleanup:
+//    i. Find all the PERSONs that have expired cookie
+//    ii. For each of their PERSON_URLs:
+//     - decrement the linked MACHINE links
+//     - if the linked MACHINE has no other links, remove the linked MACHINE too
+//     - remove the PERSON_URL
+//
+// 2. add protection to prevent abusing the registry by flooding it with
+//    requests to fill the memory and crash it.
+//
+//    Possible protections:
+//    - limit the number of URLs per person
+//    - limit the number of URLs per machine
+//    - limit the number of persons
+//    - limit the number of machines
+//    - [DONE] limit the size of URLs
+//    - [DONE] limit the size of PERSON_URL names
+//    - limit the number of requests that add data to the registry,
+//      per client IP per hour
+//
+// 3. lower memory requirements
+//
+//    - embed avl structures directly into registry objects, instead of DICTIONARY
+//      [DONE for PERSON_URLs, PENDING for MACHINE_URLs]
+//    - store GUIDs in memory as UUID instead of char *
+//    - do not track persons using the demo machines only
+//      (i.e. start tracking them only when they access a non-demo machine)
+//    - [DONE] do not track custom dashboards by default
+
+#ifndef NETDATA_REGISTRY_H
+#define NETDATA_REGISTRY_H 1
+
+#include "../daemon/common.h"
+
+#define NETDATA_REGISTRY_COOKIE_NAME "netdata_registry_id"
+
+// initialize the registry
+// should only happen when netdata starts
+extern int registry_init(void);
+
+// free all data held by the registry
+// should only happen when netdata exits
+extern void registry_free(void);
+
+// HTTP requests handled by the registry
+extern int registry_request_access_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when);
+extern int registry_request_delete_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);
+extern int registry_request_search_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when);
+extern int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when);
+extern int registry_request_hello_json(RRDHOST *host, struct web_client *w);
+
+// update the registry monitoring charts
+extern void registry_statistics(void);
+
+extern char *registry_get_this_machine_guid(void);
+extern char *registry_get_this_machine_hostname(void);
+
+extern int regenerate_guid(const char *guid, char *result);
+
+#endif /* NETDATA_REGISTRY_H */
diff --git a/registry/registry_db.c b/registry/registry_db.c
new file mode 100644
index 0000000000..d8e2bbd8dd
--- /dev/null
+++ b/registry/registry_db.c
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../daemon/common.h"
+#include "registry_internals.h"
+
+int registry_db_should_be_saved(void) {
+    debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries);
+    return registry.log_count > registry.save_registry_every_entries;
+}
+
+// ----------------------------------------------------------------------------
+// INTERNAL FUNCTIONS FOR SAVING REGISTRY OBJECTS
+
+static int registry_machine_save_url(void *entry, void *file) {
+    REGISTRY_MACHINE_URL *mu = entry;
+    FILE *fp = file;
+
+    debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);
+
+    int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",
+            mu->first_t,
+            mu->last_t,
+            mu->usages,
+            mu->flags,
+            mu->url->url
+    );
+
+    // error handling is done at registry_db_save()
+
+    return ret;
+}
+
+static int registry_machine_save(void *entry, void *file) {
+    REGISTRY_MACHINE *m = entry;
+    FILE *fp = file;
+
+    debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);
+
+    int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",
+            m->first_t,
+            m->last_t,
+            m->usages,
+            m->guid
+    );
+
+    if(ret >= 0) {
+        int ret2 = dictionary_get_all(m->machine_urls, registry_machine_save_url, fp);
+        if(ret2 < 0) return ret2;
+        ret += ret2;
+    }
+
+    // error handling is done at registry_db_save()
+
+    return ret;
+}
+
+static inline int registry_person_save_url(void *entry, void *file) {
+    REGISTRY_PERSON_URL *pu = entry;
+    FILE *fp = file;
+
+    debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);
+
+    int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",
+            pu->first_t,
+            pu->last_t,
+            pu->usages,
+            pu->flags,
+            pu->machine->guid,
+            pu->machine_name,
+            pu->url->url
+    );
+
+    // error handling is done at registry_db_save()
+
+    return ret;
+}
+
+static inline int registry_person_save(void *entry, void *file) {
+    REGISTRY_PERSON *p = entry;
+    FILE *fp = file;
+
+    debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);
+
+    int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",
+            p->first_t,
+            p->last_t,
+            p->usages,
+            p->guid
+    );
+
+    if(ret >= 0) {
+        //int ret2 = dictionary_get_all(p->person_urls, registry_person_save_url, fp);
+        int ret2 = avl_traverse(&p->person_urls, registry_person_save_url, fp);
+        if (ret2 < 0) return ret2;
+        ret += ret2;
+    }
+
+    // error handling is done at registry_db_save()
+
+    return ret;
+}
+
+// ----------------------------------------------------------------------------
+// SAVE THE REGISTRY DATABASE
+
+int registry_db_save(void) {
+    if(unlikely(!registry.enabled))
+        return -1;
+
+    if(unlikely(!registry_db_should_be_saved()))
+        return -2;
+
+    error_log_limit_unlimited();
+
+    char tmp_filename[FILENAME_MAX + 1];
+    char old_filename[FILENAME_MAX + 1];
+
+    snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);
+    snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);
+
+    debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);
+    FILE *fp = fopen(tmp_filename, "w");
+    if(!fp) {
+        error("Registry: Cannot create file: %s", tmp_filename);
+        error_log_limit_reset();
+        return -1;
+    }
+
+    // dictionary_get_all() has its own locking, so this is safe to do
+
+    debug(D_REGISTRY, "Saving all machines");
+    int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);
+    if(bytes1 < 0) {
+        error("Registry: Cannot save registry machines - return value %d", bytes1);
+        fclose(fp);
+        error_log_limit_reset();
+        return bytes1;
+    }
+    debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);
+
+    debug(D_REGISTRY, "Saving all persons");
+    int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);
+    if(bytes2 < 0) {
+        error("Registry: Cannot save registry persons - return value %d", bytes2);
+        fclose(fp);
+        error_log_limit_reset();
+        return bytes2;
+    }
+    debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);
+
+    // save the totals
+    fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",
+            registry.persons_count,
+            registry.machines_count,
+            registry.usages_count + 1, // this is required - it is lost on db rotation
+            registry.urls_count,
+            registry.persons_urls_count,
+            registry.machines_urls_count
+    );
+
+    fclose(fp);
+
+    errno = 0;
+
+    // remove the .old db
+    debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);
+    if(unlink(old_filename) == -1 && errno != ENOENT)
+        error("Registry: cannot remove old registry file '%s'", old_filename);
+
+    // rename the db to .old
+    debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);
+    if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)
+        error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", registry.db_filename, old_filename);
+
+    else {
+        // remove the database (it is saved in .old)
+        debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);
+        if (unlink(registry.db_filename) == -1 && errno != ENOENT)
+            error("Registry: cannot remove old registry file '%s'", registry.db_filename);
+
+        // move the .tmp to make it active
+        debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);
+        if (link(tmp_filename, registry.db_filename) == -1) {
+            error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,
+                    registry.db_filename);
+
+            // move the .old back
+            debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);
+            if(link(old_filename, registry.db_filename) == -1)
+                error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);
+        }
+        else {
+            debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);
+            if(unlink(tmp_filename) == -1)
+                error("Registry: cannot remove tmp registry file '%s'", tmp_filename);
+
+            // it has been moved successfully
+            // discard the current registry log
+            registry_log_recreate();
+            registry.log_count = 0;
+        }
+    }
+
+    // continue operations
+    error_log_limit_reset();
+
+    return -1;
+}
+
+// ----------------------------------------------------------------------------
+// LOAD THE REGISTRY DATABASE
+
+size_t registry_db_load(void) {
+    char *s, buf[4096 + 1];
+    REGISTRY_PERSON *p = NULL;
+    REGISTRY_MACHINE *m = NULL;
+    REGISTRY_URL *u = NULL;
+    size_t line = 0;
+
+    debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);
+    FILE *fp = fopen(registry.db_filename, "r");
+    if(!fp) {
+        error("Registry: cannot open registry file: '%s'", registry.db_filename);
+        return 0;
+    }
+
+    size_t len = 0;
+    buf[4096] = '\0';
+    while((s = fgets_trim_len(buf, 4096, fp, &len))) {
+        line++;
+
+        debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);
+        switch(*s) {
+            case 'T': // totals
+                if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {
+                    error("Registry totals line %zu is wrong (len = %zu).", line, len);
+                    continue;
+                }
+                registry.persons_count = strtoull(&s[2], NULL, 16);
+                registry.machines_count = strtoull(&s[19], NULL, 16);
+                registry.usages_count = strtoull(&s[36], NULL, 16);
+                registry.urls_count = strtoull(&s[53], NULL, 16);
+                registry.persons_urls_count = strtoull(&s[70], NULL, 16);
+                registry.machines_urls_count = strtoull(&s[87], NULL, 16);
+                break;
+
+            case 'P': // person
+                m = NULL;
+                // verify it is valid
+                if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
+                    error("Registry person line %zu is wrong (len = %zu).", line, len);
+                    continue;
+                }
+
+                s[1] = s[10] = s[19] = s[28] = '\0';
+                p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));
+                p->last_t = (uint32_t)strtoul(&s[11], NULL, 16);
+                p->usages = (uint32_t)strtoul(&s[20], NULL, 16);
+                debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);
+                break;
+
+            case 'M': // machine
+                p = NULL;
+                // verify it is valid
+                if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
+                    error("Registry person line %zu is wrong (len = %zu).", line, len);
+                    continue;
+                }
+
+                s[1] = s[10] = s[19] = s[28] = '\0';
+                m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));
+                m->last_t = (uint32_t)strtoul(&s[11], NULL, 16);
+                m->usages = (uint32_t)strtoul(&s[20], NULL, 16);
+                debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);
+                break;
+
+            case 'U': // person URL
+                if(unlikely(!p)) {
+                    error("Registry: ignoring line %zu, no person loaded: %s", line, s);
+                    continue;
+                }
+
+                // verify it is valid
+                if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {
+                    error("Registry person URL line %zu is wrong (len = %zu).", line, len);
+                    continue;
+                }
+
+                s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';
+
+                // skip the name to find the url
+                char *url = &s[69];
+                while(*url && *url != '\t'