Add a plugin for the system clock synchronization state (#10895)

Co-authored-by: Joel Hans <joel.g.hans@gmail.com>

15 files changed, 304 insertions, 32 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5088d1381e..fe125e5728 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -458,6 +458,11 @@ set(DISKSPACE_PLUGIN_FILES
         collectors/diskspace.plugin/plugin_diskspace.c
         )
 
+set(TIMEX_PLUGIN_FILES
+        collectors/timex.plugin/plugin_timex.h
+        collectors/timex.plugin/plugin_timex.c
+        )
+
 set(FREEIPMI_PLUGIN_FILES
         collectors/freeipmi.plugin/freeipmi_plugin.c
         )
@@ -984,6 +989,7 @@ IF(LINUX)
     add_executable(netdata config.h ${NETDATA_FILES}
             ${CGROUPS_PLUGIN_FILES}
             ${DISKSPACE_PLUGIN_FILES}
+            ${TIMEX_PLUGIN_FILES}
             ${PROC_PLUGIN_FILES}
             ${TC_PLUGIN_FILES}
             )
diff --git a/Makefile.am b/Makefile.am
index 8355161bb6..59ab8e5a49 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -258,6 +258,11 @@ DISKSPACE_PLUGIN_FILES = \
     collectors/diskspace.plugin/plugin_diskspace.c \
     $(NULL)
 
+TIMEX_PLUGIN_FILES = \
+    collectors/timex.plugin/plugin_timex.h \
+    collectors/timex.plugin/plugin_timex.c \
+    $(NULL)
+
 FREEIPMI_PLUGIN_FILES = \
     collectors/freeipmi.plugin/freeipmi_plugin.c \
     $(LIBNETDATA_FILES) \
@@ -714,6 +719,7 @@ if LINUX
     NETDATA_FILES += \
         $(CGROUPS_PLUGIN_FILES) \
         $(DISKSPACE_PLUGIN_FILES) \
+        $(TIMEX_PLUGIN_FILES) \
         $(PROC_PLUGIN_FILES) \
         $(TC_PLUGIN_FILES) \
         $(NULL)
diff --git a/collectors/COLLECTORS.md b/collectors/COLLECTORS.md
index a795de1002..a02a027dee 100644
--- a/collectors/COLLECTORS.md
+++ b/collectors/COLLECTORS.md
@@ -26,35 +26,38 @@ issues](https://github.com/netdata/netdata/issues). Use the search bar to look f
 collector—we may be looking for contributions from users such as yourself! If you don't see the collector there, make a
 [feature request](https://community.netdata.cloud/c/feature-requests/7/none) on our community forums.
 
--   [Service and application collectors](#service-and-application-collectors)
-    -   [APM (application performance monitoring)](#apm-application-performance-monitoring)
-    -   [Containers and VMs](#containers-and-vms)
-    -   [Data stores](#data-stores)
-    -   [Distributed computing](#distributed-computing)
-    -   [Email](#email)
-    -   [Kubernetes](#kubernetes)
-    -   [Logs](#logs)
-    -   [Messaging](#messaging)
-    -   [Network](#network)
-    -   [Provisioning](#provisioning)
-    -   [Remote devices](#remote-devices)
-    -   [Search](#search)
-    -   [Storage](#storage)
-    -   [Web](#web)
--   [System collectors](#system-collectors)
-    -   [Applications](#applications)
-    -   [Disks and filesystems](#disks-and-filesystems)
-    -   [eBPF (extended Berkeley Packet Filter)](#ebpf)
-    -   [Hardware](#hardware)
-    -   [Memory](#memory)
-    -   [Networks](#networks)
-    -   [Processes](#processes)
-    -   [Resources](#resources)
-    -   [Users](#users)
--   [Netdata collectors](#netdata-collectors)
--   [Orchestrators](#orchestrators)
--   [Third-party collectors](#third-party-collectors)
--   [Etc](#etc)
+- [Supported collectors list](#supported-collectors-list)
+  - [Service and application collectors](#service-and-application-collectors)
+    - [Generic](#generic)
+    - [APM (application performance monitoring)](#apm-application-performance-monitoring)
+    - [Containers and VMs](#containers-and-vms)
+    - [Data stores](#data-stores)
+    - [Distributed computing](#distributed-computing)
+    - [Email](#email)
+    - [Kubernetes](#kubernetes)
+    - [Logs](#logs)
+    - [Messaging](#messaging)
+    - [Network](#network)
+    - [Provisioning](#provisioning)
+    - [Remote devices](#remote-devices)
+    - [Search](#search)
+    - [Storage](#storage)
+    - [Web](#web)
+  - [System collectors](#system-collectors)
+    - [Applications](#applications)
+    - [Disks and filesystems](#disks-and-filesystems)
+    - [eBPF](#ebpf)
+    - [Hardware](#hardware)
+    - [Memory](#memory)
+    - [Networks](#networks)
+    - [Operating systems](#operating-systems)
+    - [Processes](#processes)
+    - [Resources](#resources)
+    - [Users](#users)
+  - [Netdata collectors](#netdata-collectors)
+  - [Orchestrators](#orchestrators)
+  - [Third-party collectors](#third-party-collectors)
+  - [Etc](#etc)
 
 ## Service and application collectors
 
@@ -366,6 +369,7 @@ The Netdata Agent can collect these system- and hardware-level metrics using a v
 -   [Device mapper](/collectors/proc.plugin/README.md): Gather metrics about the Linux device mapper with the proc
     collector.
 -   [Disk space](/collectors/diskspace.plugin/README.md): Collect disk space usage metrics on Linux mount points.
+-   [Clock synchronization](/collectors/timex.plugin/README.md): Collect the system clock synchronization status on Linux.
 -   [Files and directories](https://learn.netdata.cloud/docs/agent/collectors/go.d.plugin/modules/filecheck): Gather
     metrics about the existence, modification time, and size of files or directories.
 -   [ioping.plugin](/collectors/ioping.plugin/README.md): Measure disk read/write latency.
diff --git a/collectors/Makefile.am b/collectors/Makefile.am
index 460612c685..021e2ff23c 100644
--- a/collectors/Makefile.am
+++ b/collectors/Makefile.am
@@ -10,6 +10,7 @@ SUBDIRS = \
     checks.plugin \
     cups.plugin \
     diskspace.plugin \
+    timex.plugin \
     fping.plugin \
     ioping.plugin \
     freebsd.plugin \
diff --git a/collectors/REFERENCE.md b/collectors/REFERENCE.md
index 9c6f0a61ed..4922ff9a9d 100644
--- a/collectors/REFERENCE.md
+++ b/collectors/REFERENCE.md
@@ -96,6 +96,7 @@ This section features a list of Netdata's plugins, with a boolean setting to ena
 	# PYTHONPATH environment variable = 
 	# proc = yes
 	# diskspace = yes
+	# timex = yes
 	# cgroups = yes
 	# tc = yes
 	# idlejitter = yes
diff --git a/collectors/all.h b/collectors/all.h
index d115fdc73c..ccf89b547f 100644
--- a/collectors/all.h
+++ b/collectors/all.h
@@ -12,6 +12,7 @@
 #include "idlejitter.plugin/plugin_idlejitter.h"
 #include "cgroups.plugin/sys_fs_cgroup.h"
 #include "diskspace.plugin/plugin_diskspace.h"
+#include "timex.plugin/plugin_timex.h"
 #include "proc.plugin/plugin_proc.h"
 #include "tc.plugin/plugin_tc.h"
 #include "macos.plugin/plugin_macos.h"
@@ -53,6 +54,8 @@
 #define NETDATA_CHART_PRIO_SYSTEM_SOFT_INTR           1100 // freebsd only
 #define NETDATA_CHART_PRIO_SYSTEM_ENTROPY             1000
 #define NETDATA_CHART_PRIO_SYSTEM_UPTIME              1000
+#define NETDATA_CHART_PRIO_CLOCK_SYNC_STATE           1100
+#define NETDATA_CHART_PRIO_CLOCK_SYNC_OFFSET          1110
 #define NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_QUEUES      1200 // freebsd only
 #define NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_MESSAGES    1201
 #define NETDATA_CHART_PRIO_SYSTEM_IPC_MSQ_SIZE        1202
@@ -339,6 +342,7 @@
 #define NETDATA_CHART_PRIO_CHECKS                    99999
 
 #define NETDATA_CHART_PRIO_NETDATA_DISKSPACE        132020
+#define NETDATA_CHART_PRIO_NETDATA_TIMEX            132030
 #define NETDATA_CHART_PRIO_NETDATA_TC_CPU           135000
 #define NETDATA_CHART_PRIO_NETDATA_TC_TIME          135001
 
diff --git a/collectors/timex.plugin/Makefile.am b/collectors/timex.plugin/Makefile.am
new file mode 100644
index 0000000000..161784b8f6
--- /dev/null
+++ b/collectors/timex.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+    README.md \
+    $(NULL)
diff --git a/collectors/timex.plugin/README.md b/collectors/timex.plugin/README.md
new file mode 100644
index 0000000000..79947441f9
--- /dev/null
+++ b/collectors/timex.plugin/README.md
@@ -0,0 +1,29 @@
+<!--
+title: "timex.plugin"
+description: "Monitor the system clock synchronization state."
+custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/timex.plugin/README.md
+-->
+
+# timex.plugin
+
+This plugin monitors the system clock synchronization state on Linux nodes.
+
+This plugin creates two charts:
+
+-   System clock synchronization state
+-   Computed time offset between local system and reference clock
+
+## Configuration
+
+Edit the `netdata.conf` configuration file using [`edit-config`](/docs/configure/nodes.md#use-edit-config-to-edit-configuration-files) from the [Netdata config directory](/docs/configure/nodes.md#the-netdata-config-directory), which is typically at `/etc/netdata`.
+
+Scroll down to the `[plugin:timex]` section to find the available options:
+
+```
+[plugin:timex]
+    # update every = 1
+    # clock synchronization state = yes
+    # time offset = yes
+```
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fcollectors%2Ftimex.plugin%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)
diff --git a/collectors/timex.plugin/plugin_timex.c b/collectors/timex.plugin/plugin_timex.c
new file mode 100644
index 0000000000..0032b3ddaf
--- /dev/null
+++ b/collectors/timex.plugin/plugin_timex.c
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "plugin_timex.h"
+#include "sys/timex.h"
+
+#define PLUGIN_TIMEX_NAME "timex.plugin"
+
+#define CONFIG_SECTION_TIMEX "plugin:timex"
+
+static void timex_main_cleanup(void *ptr)
+{
+    struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
+    static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
+
+    info("cleaning up...");
+
+    static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
+}
+
+void *timex_main(void *ptr)
+{
+    netdata_thread_cleanup_push(timex_main_cleanup, ptr);
+
+    int vdo_cpu_netdata = config_get_boolean(CONFIG_SECTION_TIMEX, "timex plugin resource charts", CONFIG_BOOLEAN_YES);
+
+    int update_every = (int)config_get_number(CONFIG_SECTION_TIMEX, "update every", 10);
+    if (update_every < localhost->rrd_update_every)
+        update_every = localhost->rrd_update_every;
+
+    int do_sync = config_get_boolean(CONFIG_SECTION_TIMEX, "clock synchronization state", CONFIG_BOOLEAN_YES);
+    int do_offset = config_get_boolean(CONFIG_SECTION_TIMEX, "time offset", CONFIG_BOOLEAN_YES);
+
+    if (unlikely(do_sync == CONFIG_BOOLEAN_NO && do_offset == CONFIG_BOOLEAN_NO)) {
+        info("No charts to show");
+        goto exit;
+    }
+
+    usec_t step = update_every * USEC_PER_SEC;
+    heartbeat_t hb;
+    heartbeat_init(&hb);
+    while (!netdata_exit) {
+        usec_t duration = heartbeat_monotonic_dt_to_now_usec(&hb);
+        heartbeat_next(&hb, step);
+
+        struct timex timex_buf = {};
+        int sync_state = 0;
+        sync_state = adjtimex(&timex_buf);
+
+        collected_number divisor = USEC_PER_MS;
+        if (timex_buf.status & STA_NANO)
+            divisor = NSEC_PER_MSEC;
+
+        // ----------------------------------------------------------------
+
+        if (do_sync) {
+            static RRDSET *st_sync_state = NULL;
+            static RRDDIM *rd_sync_state;
+
+            if (unlikely(!st_sync_state)) {
+                st_sync_state = rrdset_create_localhost(
+                    "system",
+                    "clock_sync_state",
+                    NULL,
+                    "clock synchronization",
+                    NULL,
+                    "System Clock Synchronization State",
+                    "state",
+                    PLUGIN_TIMEX_NAME,
+                    NULL,
+                    NETDATA_CHART_PRIO_CLOCK_SYNC_STATE,
+                    update_every,
+                    RRDSET_TYPE_LINE);
+
+                rd_sync_state = rrddim_add(st_sync_state, "state", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
+            } else {
+                rrdset_next(st_sync_state);
+            }
+
+            rrddim_set_by_pointer(st_sync_state, rd_sync_state, sync_state != TIME_ERROR ? 1 : 0);
+            rrdset_done(st_sync_state);
+        }
+
+        if (do_sync) {
+            static RRDSET *st_offset = NULL;
+            static RRDDIM *rd_offset;
+
+            if (unlikely(!st_offset)) {
+                st_offset = rrdset_create_localhost(
+                    "system",
+                    "clock_sync_offset",
+                    NULL,
+                    "clock synchronization",
+                    NULL,
+                    "Computed Time Offset Between Local System and Reference Clock",
+                    "milliseconds",
+                    PLUGIN_TIMEX_NAME,
+                    NULL,
+                    NETDATA_CHART_PRIO_CLOCK_SYNC_OFFSET,
+                    update_every,
+                    RRDSET_TYPE_LINE);
+
+                rd_offset = rrddim_add(st_offset, "offset", NULL, 1, divisor, RRD_ALGORITHM_ABSOLUTE);
+            } else {
+                rrdset_next(st_offset);
+            }
+
+            rrddim_set_by_pointer(st_offset, rd_offset, timex_buf.offset);
+            rrdset_done(st_offset);
+        }
+
+        if (vdo_cpu_netdata) {
+            static RRDSET *stcpu_thread = NULL, *st_duration = NULL;
+            static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL;
+
+            // ----------------------------------------------------------------
+
+            struct rusage thread;
+            getrusage(RUSAGE_THREAD, &thread);
+
+            if (unlikely(!stcpu_thread)) {
+                stcpu_thread = rrdset_create_localhost(
+                    "netdata",
+                    "plugin_timex",
+                    NULL,
+                    "timex",
+                    NULL,
+                    "NetData Timex Plugin CPU usage",
+                    "milliseconds/s",
+                    PLUGIN_TIMEX_NAME,
+                    NULL,
+                    NETDATA_CHART_PRIO_NETDATA_TIMEX,
+                    update_every,
+                    RRDSET_TYPE_STACKED);
+
+                rd_user   = rrddim_add(stcpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
+                rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
+            } else {
+                rrdset_next(stcpu_thread);
+            }
+
+            rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
+            rrddim_set_by_pointer(
+                stcpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
+            rrdset_done(stcpu_thread);
+
+            // ----------------------------------------------------------------
+
+            if (unlikely(!st_duration)) {
+                st_duration = rrdset_create_localhost(
+                    "netdata",
+                    "plugin_timex_dt",
+                    NULL,
+                    "timex",
+                    NULL,
+                    "NetData Timex Plugin Duration",
+                    "milliseconds/run",
+                    PLUGIN_TIMEX_NAME,
+                    NULL,
+                    NETDATA_CHART_PRIO_NETDATA_TIMEX + 1,
+                    update_every,
+                    RRDSET_TYPE_AREA);
+
+                rd_duration = rrddim_add(st_duration, "duration", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE);
+            } else {
+                rrdset_next(st_duration);
+            }
+
+            rrddim_set_by_pointer(st_duration, rd_duration, duration);
+            rrdset_done(st_duration);
+        }
+    }
+
+exit:
+    netdata_thread_cleanup_pop(1);
+    return NULL;
+}
diff --git a/collectors/timex.plugin/plugin_timex.h b/collectors/timex.plugin/plugin_timex.h
new file mode 100644
index 0000000000..6025641a35
--- /dev/null
+++ b/collectors/timex.plugin/plugin_timex.h
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_PLUGIN_TIMEX_H
+#define NETDATA_PLUGIN_TIMEX_H
+
+#include "../../daemon/common.h"
+
+#if (TARGET_OS == OS_LINUX)
+
+#define NETDATA_PLUGIN_HOOK_LINUX_TIMEX \
+    { \
+        .name = "PLUGIN[timex]", \
+        .config_section = CONFIG_SECTION_PLUGINS, \
+        .config_name = "timex", \
+        .enabled = 1, \
+        .thread = NULL, \
+        .init_routine = NULL, \
+        .start_routine = timex_main \
+    },
+
+extern void *timex_main(void *ptr);
+
+#else // (TARGET_OS == OS_LINUX)
+
+#define NETDATA_PLUGIN_HOOK_LINUX_TIMEX
+
+#endif // (TARGET_OS == OS_LINUX)
+
+#endif //NETDATA_PLUGIN_TIMEX_H
diff --git a/configure.ac b/configure.ac
index b42b007de3..03672c5518 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1561,6 +1561,7 @@ AC_CONFIG_FILES([
     collectors/charts.d.plugin/Makefile
     collectors/checks.plugin/Makefile
     collectors/diskspace.plugin/Makefile
+    collectors/timex.plugin/Makefile
     collectors/fping.plugin/Makefile
     collectors/ioping.plugin/Makefile
     collectors/freebsd.plugin/Makefile
diff --git a/daemon/main.c b/daemon/main.c
index 93ec78b6ab..697f7256ff 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -79,6 +79,7 @@ struct netdata_static_thread static_threads[] = {
     // linux internal plugins
     NETDATA_PLUGIN_HOOK_LINUX_PROC
     NETDATA_PLUGIN_HOOK_LINUX_DISKSPACE
+    NETDATA_PLUGIN_HOOK_LINUX_TIMEX
     NETDATA_PLUGIN_HOOK_LINUX_CGROUPS
     NETDATA_PLUGIN_HOOK_LINUX_TC
 
diff --git a/tests/alarm_repetition/netdata.conf_with_repetition b/tests/alarm_repetition/netdata.conf_with_repetition
index d5d00f07da..808a11c560 100644
--- a/tests/alarm_repetition/netdata.conf_with_repetition
+++ b/tests/alarm_repetition/netdata.conf_with_repetition
@@ -29,6 +29,7 @@
 [plugins]
     proc = yes
     diskspace = no
+    timex = no
     cgroups = no
     tc = no
     idlejitter = no
diff --git a/tests/alarm_repetition/netdata.conf_without_repetition b/tests/alarm_repetition/netdata.conf_without_repetition
index 43518bdc00..978db8d205 100644
--- a/tests/alarm_repetition/netdata.conf_without_repetition
+++ b/tests/alarm_repetition/netdata.conf_without_repetition
@@ -29,6 +29,7 @@
 [plugins]
     proc = yes
     diskspace = no
+    timex = no
     cgroups = no
     tc = no
     idlejitter = no
diff --git a/web/gui/dashboard_info.js b/web/gui/dashboard_info.js
index 81a7827044..bda57c0c40 100644
--- a/web/gui/dashboard_info.js
+++ b/web/gui/dashboard_info.js
@@ -842,6 +842,10 @@ netdataDashboard.context = {
         info: '<a href="https://en.wikipedia.org/wiki/Entropy_(computing)" target="_blank">Entropy</a>, is a pool of random numbers (<a href="https://en.wikipedia.org/wiki//dev/random" target="_blank">/dev/random</a>) that is mainly used in cryptography. If the pool of entropy gets empty, processes requiring random numbers may run a lot slower (it depends on the interface each program uses), waiting for the pool to be replenished. Ideally a system with high entropy demands should have a hardware device for that purpose (TPM is one such device). There are also several software-only options you may install, like <code>haveged</code>, although these are generally useful only in servers.'
     },
 
+    'system.clock_sync_state': {
+        info: 'State map: 0 - not synchronized, 1 - synchronized'
+    },
+
     'system.forks': {
         colors: '#5555DD',
         info: 'Number of new processes created.'
@@ -1461,13 +1465,13 @@ netdataDashboard.context = {
         info: 'inodes (or index nodes) are filesystem objects (e.g. files and directories). On many types of file system implementations, the maximum number of inodes is fixed at filesystem creation, limiting the maximum number of files the filesystem can hold. It is possible for a device to run out of inodes. When this happens, new files cannot be created on the device, even though there may be free space available.'
     },
 
+    // ------------------------------------------------------------------------
+    // MYSQL
+
     'mysql.net': {
         info: 'The amount of data sent to mysql clients (<strong>out</strong>) and received from mysql clients (<strong>in</strong>).'
     },
 
-    // ------------------------------------------------------------------------
-    // MYSQL
-
     'mysql.queries': {
         info: 'The number of statements executed by the server.<ul>' +
             '<li><strong>queries</strong> counts the statements executed within stored SQL programs.</li>' +