Perf plugin (#6225)

* Add perf plugin skeleton

* Initialize events

* Collect data

* Configure default counters

* Add charts for hardware and software counters

* Add charts for cache counters

* Don't show zeroes for non-existent metrics

* Reinit events when stalled

* Do not reinit disabled events

* Update the documentation

* Scale values when multiplexing is happening

15 files changed, 1515 insertions, 15 deletions

diff --git a/.gitignore b/.gitignore
index f393adbfd7..789043e744 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,6 +56,9 @@ nfacct.plugin
 xenstat.plugin
 !xenstat.plugin/
 
+perf.plugin
+!perf.plugin/
+
 cgroup-network
 !cgroup-network/
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 690c1272bf..e6109c2fc2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -370,6 +370,10 @@ set(XENSTAT_PLUGIN_FILES
         collectors/xenstat.plugin/xenstat_plugin.c
         )
 
+set(PERF_PLUGIN_FILES
+        collectors/perf.plugin/perf_plugin.c
+        )
+
 set(PROC_PLUGIN_FILES
         collectors/proc.plugin/ipc.c
         collectors/proc.plugin/plugin_proc.c
@@ -678,6 +682,7 @@ IF(LINUX)
 
     SET(ENABLE_PLUGIN_CGROUP_NETWORK True)
     SET(ENABLE_PLUGIN_APPS True)
+    SET(ENABLE_PLUGIN_PERF True)
 
 ELSEIF(FREEBSD)
     add_executable(netdata config.h ${NETDATA_FILES} ${FREEBSD_PLUGIN_FILES})
@@ -686,6 +691,7 @@ ELSEIF(FREEBSD)
     target_compile_options(netdata PUBLIC ${NETDATA_COMMON_CFLAGS})
     SET(ENABLE_PLUGIN_CGROUP_NETWORK False)
     SET(ENABLE_PLUGIN_APPS True)
+    SET(ENABLE_PLUGIN_PERF False)
 
 ELSEIF(MACOS)
     add_executable(netdata config.h ${NETDATA_FILES} ${MACOS_PLUGIN_FILES})
@@ -694,6 +700,7 @@ ELSEIF(MACOS)
     target_compile_options(netdata PUBLIC ${NETDATA_COMMON_CFLAGS})
     SET(ENABLE_PLUGIN_CGROUP_NETWORK False)
     SET(ENABLE_PLUGIN_APPS False)
+    SET(ENABLE_PLUGIN_PERF False)
 
 ENDIF()
 
@@ -778,6 +785,20 @@ ENDIF()
 
 
 # -----------------------------------------------------------------------------
+# perf.plugin
+
+IF(ENABLE_PLUGIN_PERF)
+    message(STATUS "perf.plugin: enabled")
+    add_executable(perf.plugin config.h ${PERF_PLUGIN_FILES})
+    target_link_libraries (perf.plugin libnetdata ${NETDATA_COMMON_LIBRARIES})
+    target_include_directories(perf.plugin PUBLIC ${NETDATA_COMMON_INCLUDE_DIRS})
+    target_compile_options(perf.plugin PUBLIC ${NETDATA_COMMON_CFLAGS})
+ELSE()
+    message(STATUS "perf.plugin: disabled")
+ENDIF()
+
+
+# -----------------------------------------------------------------------------
 # cgroup-network
 
 IF(ENABLE_PLUGIN_CGROUP_NETWORK)
diff --git a/Makefile.am b/Makefile.am
index d5a873dfd5..6dacd825ef 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -234,6 +234,11 @@ XENSTAT_PLUGIN_FILES = \
     $(LIBNETDATA_FILES) \
     $(NULL)
 
+PERF_PLUGIN_FILES = \
+    collectors/perf.plugin/perf_plugin.c \
+    $(LIBNETDATA_FILES) \
+    $(NULL)
+
 PROC_PLUGIN_FILES = \
 	collectors/proc.plugin/ipc.c \
 	collectors/proc.plugin/plugin_proc.c \
@@ -573,6 +578,14 @@ if ENABLE_PLUGIN_XENSTAT
         $(NULL)
 endif
 
+if ENABLE_PLUGIN_PERF
+    plugins_PROGRAMS += perf.plugin
+    perf_plugin_SOURCES = $(PERF_PLUGIN_FILES)
+    perf_plugin_LDADD = \
+        $(NETDATA_COMMON_LIBS) \
+        $(NULL)
+endif
+
 if ENABLE_BACKEND_KINESIS
     netdata_SOURCES += $(KINESIS_BACKEND_FILES)
     netdata_LDADD += $(OPTIONAL_KINESIS_LIBS)
diff --git a/collectors/Makefile.am b/collectors/Makefile.am
index 87a037e762..fe62ba01d2 100644
--- a/collectors/Makefile.am
+++ b/collectors/Makefile.am
@@ -18,6 +18,7 @@ SUBDIRS = \
     macos.plugin \
     nfacct.plugin \
     xenstat.plugin \
+    perf.plugin \
     node.d.plugin \
     proc.plugin \
     python.d.plugin \
diff --git a/collectors/README.md b/collectors/README.md
index 154d193e49..7252138893 100644
--- a/collectors/README.md
+++ b/collectors/README.md
@@ -37,6 +37,7 @@ plugin|lang|O/S|runs as|modular|description
 [macos.plugin](macos.plugin/)|`C`|macos|internal|yes|collects resource usage and performance data on MacOS systems
 [nfacct.plugin](nfacct.plugin/)|`C`|linux|external|-|collects netfilter firewall, connection tracker and accounting metrics using `libmnl` and `libnetfilter_acct`
 [xenstat.plugin](xenstat.plugin/)|`C`|linux|external|-|collects XenServer and XCP-ng metrics using `libxenstat`
+[perf.plugin](perf.plugin/)|`C`|linux|external|-|collects CPU performance metrics using performance monitoring units (PMU).
 [node.d.plugin](node.d.plugin/)|`node.js`|any|external|yes|a **plugin orchestrator** for data collection modules written in `node.js`.
 [plugins.d](plugins.d/)|`C`|any|internal|-|implements the **external plugins** API and serves external plugins
 [proc.plugin](proc.plugin/)|`C`|linux|internal|yes|collects resource usage and performance data on Linux systems
diff --git a/collectors/apps.plugin/apps_groups.conf b/collectors/apps.plugin/apps_groups.conf
index ac72fffdb2..c4875bd6ba 100644
--- a/collectors/apps.plugin/apps_groups.conf
+++ b/collectors/apps.plugin/apps_groups.conf
@@ -77,6 +77,7 @@ freeipmi.plugin: freeipmi.plugin
 nfacct.plugin: nfacct.plugin
 cups.plugin: cups.plugin
 xenstat.plugin: xenstat.plugin
+perf.plugin: perf.plugin
 charts.d.plugin: *charts.d.plugin*
 node.d.plugin: *node.d.plugin*
 python.d.plugin: *python.d.plugin*
diff --git a/collectors/perf.plugin/Makefile.am b/collectors/perf.plugin/Makefile.am
new file mode 100644
index 0000000000..19554bed8e
--- /dev/null
+++ b/collectors/perf.plugin/Makefile.am
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+AUTOMAKE_OPTIONS = subdir-objects
+MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
+
+dist_noinst_DATA = \
+	README.md \
+	$(NULL)
diff --git a/collectors/perf.plugin/README.md b/collectors/perf.plugin/README.md
new file mode 100644
index 0000000000..ce696b06d3
--- /dev/null
+++ b/collectors/perf.plugin/README.md
@@ -0,0 +1,72 @@
+# perf.plugin
+
+`perf.plugin` collects system-wide CPU performance statistics from Performance Monitoring Units (PMU) using
+the `perf_event_open()` system call.
+
+## Important Notes
+
+Accessing hardware PMUs requires root permissions, so the plugin is setuid to root.
+
+Keep in mind that the number of PMUs in a system is usually quite limited and every hardware monitoring
+event for every CPU core needs a separate file descriptor to be opened.
+
+## Charts
+
+The plugin provides statistics for general hardware and software performance monitoring events:
+
+Hardware events:
+1.  CPU cycles
+2.  Instructions
+3.  Branch instructions
+4.  Cache operations
+5.  BUS cycles
+6.  Stalled frontend and backend cycles
+
+Software events:
+1.  CPU migrations
+2.  Alignment faults
+3.  Emulation faults
+
+Hardware cache events:
+1.  L1D cache operations
+2.  L1D prefetch cache operations
+3.  L1I cache operations
+4.  LL cache operations
+5.  DTLB cache operations
+6.  ITLB cache operations
+7.  PBU cache operations
+
+## Configuration
+
+The plugin is disabled by default because the number of PMUs is usually quite limited and it is not desired to
+allow Netdata to struggle silently for PMUs, interfering with other performance monitoring software. If you need to
+enable the perf plugin, edit /etc/netdata/netdata.conf and set:
+
+```raw
+[plugins]
+    perf = yes
+```
+
+```raw
+[plugin:perf]
+    update every = 1
+    command options = all
+```
+
+You can use the `command options` parameter to pick what data should be collected and which charts should be
+displayed. If `all` is used, all general performance monitoring counters are probed and corresponding charts
+are enabled for the available counters. You can also define a particular set of enabled charts using the
+following keywords: `cycles`, `instructions`, `branch`, `cache`, `bus`, `stalled`, `migrations`, `alighnment`,
+`emulation`, `L1D`, `L1D-prefetch`, `L1I`, `LL`, `DTLB`, `ITLB`, `PBU`.
+
+## Debugging
+
+You can run the plugin by hand:
+
+```raw
+sudo /usr/libexec/netdata/plugins.d/perf.plugin 1 all debug
+```
+
+You will get verbose output on what the plugin does.
+
+[![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fcollectors%2Fperf.plugin%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)]()
diff --git a/collectors/perf.plugin/perf_plugin.c b/collectors/perf.plugin/perf_plugin.c
new file mode 100644
index 0000000000..c645c2798f
--- /dev/null
+++ b/collectors/perf.plugin/perf_plugin.c
@@ -0,0 +1,1348 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "../../libnetdata/libnetdata.h"
+
+#include <linux/perf_event.h>
+
+#define PLUGIN_PERF_NAME "perf.plugin"
+
+// Hardware counters
+#define NETDATA_CHART_PRIO_PERF_CPU_CYCLES            8800
+#define NETDATA_CHART_PRIO_PERF_INSTRUCTIONS          8801
+#define NETDATA_CHART_PRIO_PERF_BRANCH_INSTRUSTIONS   8802
+#define NETDATA_CHART_PRIO_PERF_CACHE                 8803
+#define NETDATA_CHART_PRIO_PERF_BUS_CYCLES            8804
+#define NETDATA_CHART_PRIO_PERF_FRONT_BACK_CYCLES     8805
+
+// Software counters
+#define NETDATA_CHART_PRIO_PERF_MIGRATIONS            8810
+#define NETDATA_CHART_PRIO_PERF_ALIGNMENT             8811
+#define NETDATA_CHART_PRIO_PERF_EMULATION             8812
+
+// Hardware cache counters
+#define NETDATA_CHART_PRIO_PERF_L1D                   8820
+#define NETDATA_CHART_PRIO_PERF_L1D_PREFETCH          8821
+#define NETDATA_CHART_PRIO_PERF_L1I                   8822
+#define NETDATA_CHART_PRIO_PERF_LL                    8823
+#define NETDATA_CHART_PRIO_PERF_DTLB                  8824
+#define NETDATA_CHART_PRIO_PERF_ITLB                  8825
+#define NETDATA_CHART_PRIO_PERF_PBU                   8826
+
+// callback required by fatal()
+void netdata_cleanup_and_exit(int ret) {
+    exit(ret);
+}
+
+void send_statistics( const char *action, const char *action_result, const char *action_data) {
+    (void) action;
+    (void) action_result;
+    (void) action_data;
+    return;
+}
+
+// callbacks required by popen()
+void signals_block(void) {};
+void signals_unblock(void) {};
+void signals_reset(void) {};
+
+// callback required by eval()
+int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result) {
+    (void)variable;
+    (void)hash;
+    (void)rc;
+    (void)result;
+    return 0;
+};
+
+// required by get_system_cpus()
+char *netdata_configured_host_prefix = "";
+
+// Variables
+
+#define RRD_TYPE_PERF "perf"
+#define RRD_FAMILY_HW "hardware"
+#define RRD_FAMILY_SW "software"
+#define RRD_FAMILY_CACHE "cache"
+
+#define NO_FD -1
+#define ALL_PIDS -1
+#define RUNNING_THRESHOLD 100
+
+static int debug = 0;
+
+static int update_every = 1;
+static int freq = 0;
+
+typedef enum perf_event_id {
+    // Hardware counters
+    EV_ID_CPU_CYCLES,
+    EV_ID_INSTRUCTIONS,
+    EV_ID_CACHE_REFERENCES,
+    EV_ID_CACHE_MISSES,
+    EV_ID_BRANCH_INSTRUCTIONS,
+    EV_ID_BRANCH_MISSES,
+    EV_ID_BUS_CYCLES,
+    EV_ID_STALLED_CYCLES_FRONTEND,
+    EV_ID_STALLED_CYCLES_BACKEND,
+    EV_ID_REF_CPU_CYCLES,
+
+    // Software counters
+    // EV_ID_CPU_CLOCK,
+    // EV_ID_TASK_CLOCK,
+    // EV_ID_PAGE_FAULTS,
+    // EV_ID_CONTEXT_SWITCHES,
+    EV_ID_CPU_MIGRATIONS,
+    // EV_ID_PAGE_FAULTS_MIN,
+    // EV_ID_PAGE_FAULTS_MAJ,
+    EV_ID_ALIGNMENT_FAULTS,
+    EV_ID_EMULATION_FAULTS,
+
+    // Hardware cache counters
+    EV_ID_L1D_READ_ACCESS,
+    EV_ID_L1D_READ_MISS,
+    EV_ID_L1D_WRITE_ACCESS,
+    EV_ID_L1D_WRITE_MISS,
+    EV_ID_L1D_PREFETCH_ACCESS,
+
+    EV_ID_L1I_READ_ACCESS,
+    EV_ID_L1I_READ_MISS,
+
+    EV_ID_LL_READ_ACCESS,
+    EV_ID_LL_READ_MISS,
+    EV_ID_LL_WRITE_ACCESS,
+    EV_ID_LL_WRITE_MISS,
+
+    EV_ID_DTLB_READ_ACCESS,
+    EV_ID_DTLB_READ_MISS,
+    EV_ID_DTLB_WRITE_ACCESS,
+    EV_ID_DTLB_WRITE_MISS,
+
+    EV_ID_ITLB_READ_ACCESS,
+    EV_ID_ITLB_READ_MISS,
+
+    EV_ID_PBU_READ_ACCESS,
+
+    EV_ID_END
+} perf_event_id_t;
+
+enum perf_event_group {
+    EV_GROUP_CYCLES,
+    EV_GROUP_INSTRUCTIONS_AND_CACHE,
+    EV_GROUP_SOFTWARE,
+    EV_GROUP_CACHE_L1D,
+    EV_GROUP_CACHE_L1I_LL_DTLB,
+    EV_GROUP_CACHE_ITLB_BPU,
+
+    EV_GROUP_NUM
+};
+
+static int number_of_cpus;
+
+static int *group_leader_fds[EV_GROUP_NUM];
+
+static struct perf_event {
+    perf_event_id_t id;
+
+    int type;
+    int config;
+
+    int **group_leader_fd;
+    int *fd;
+
+    int disabled;
+    int updated;
+
+    uint64_t value;
+
+    uint64_t *prev_value;
+    uint64_t *prev_time_enabled;
+    uint64_t *prev_time_running;
+} perf_events[] = {
+    // Hardware counters
+    {EV_ID_CPU_CYCLES,              PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES,              &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_INSTRUCTIONS,            PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS,            &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_CACHE_REFERENCES,        PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES,        &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_CACHE_MISSES,            PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES,            &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_BRANCH_INSTRUCTIONS,     PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_INSTRUCTIONS,     &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_BRANCH_MISSES,           PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_MISSES,           &group_leader_fds[EV_GROUP_INSTRUCTIONS_AND_CACHE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_BUS_CYCLES,              PERF_TYPE_HARDWARE, PERF_COUNT_HW_BUS_CYCLES,              &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_STALLED_CYCLES_FRONTEND, PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_STALLED_CYCLES_BACKEND,  PERF_TYPE_HARDWARE, PERF_COUNT_HW_STALLED_CYCLES_BACKEND,  &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_REF_CPU_CYCLES,          PERF_TYPE_HARDWARE, PERF_COUNT_HW_REF_CPU_CYCLES,          &group_leader_fds[EV_GROUP_CYCLES], NULL, 1, 0, 0, NULL, NULL, NULL},
+
+    // Software counters
+    // {EV_ID_CPU_CLOCK,        PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_CLOCK,        &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    // {EV_ID_TASK_CLOCK,       PERF_TYPE_SOFTWARE, PERF_COUNT_SW_TASK_CLOCK,       &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    // {EV_ID_PAGE_FAULTS,      PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS,      &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    // {EV_ID_CONTEXT_SWITCHES, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CONTEXT_SWITCHES, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_CPU_MIGRATIONS,   PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CPU_MIGRATIONS,   &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    // {EV_ID_PAGE_FAULTS_MIN,  PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MIN,  &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    // {EV_ID_PAGE_FAULTS_MAJ,  PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS_MAJ,  &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_ALIGNMENT_FAULTS, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_ALIGNMENT_FAULTS, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+    {EV_ID_EMULATION_FAULTS, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_EMULATION_FAULTS, &group_leader_fds[EV_GROUP_SOFTWARE], NULL, 1, 0, 0, NULL, NULL, NULL},
+
+    // Hardware cache counters
+    {
+        EV_ID_L1D_READ_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_L1D_READ_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_L1D_WRITE_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_L1D_WRITE_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_L1D_PREFETCH_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1D) | (PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1D], NULL, 1, 0, 0, NULL, NULL, NULL
+    },
+
+    {
+        EV_ID_L1I_READ_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1I) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_L1I_READ_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_L1I) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    },
+
+    {
+        EV_ID_LL_READ_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_LL_READ_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_LL_WRITE_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_LL_WRITE_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_LL) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    },
+
+    {
+        EV_ID_DTLB_READ_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_DTLB_READ_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_DTLB_WRITE_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_L1I_LL_DTLB], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_DTLB_WRITE_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_DTLB) | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+    },
+
+    {
+        EV_ID_ITLB_READ_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_ITLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+    }, {
+        EV_ID_ITLB_READ_MISS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_ITLB) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+    },
+
+    {
+        EV_ID_PBU_READ_ACCESS, PERF_TYPE_HW_CACHE,
+        (PERF_COUNT_HW_CACHE_BPU) | (PERF_COUNT_HW_CACHE_OP_READ << 8) | (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
+        &group_leader_fds[EV_GROUP_CACHE_ITLB_BPU], NULL, 1, 0, 0, NULL, NULL, NULL
+    },
+
+    {EV_ID_END, 0, 0, NULL, NULL, 0, 0, 0, NULL, NULL, NULL}
+};
+
+static int perf_init() {
+    int cpu, group;
+    struct perf_event_attr perf_event_attr;
+    struct perf_event *current_event = NULL;
+    unsigned long flags = 0;
+
+    number_of_cpus = (int)get_system_cpus();
+
+    // initialize all perf event file descriptors
+    for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+        current_event->fd = mallocz(number_of_cpus * sizeof(int));
+        memset(current_event->fd, NO_FD, number_of_cpus * sizeof(int));
+
+        current_event->prev_value = mallocz(number_of_cpus * sizeof(uint64_t));
+        memset(current_event->prev_value, 0, number_of_cpus * sizeof(uint64_t));
+
+        current_event->prev_time_enabled = mallocz(number_of_cpus * sizeof(uint64_t));
+        memset(current_event->prev_time_enabled, 0, number_of_cpus * sizeof(uint64_t));
+
+        current_event->prev_time_running = mallocz(number_of_cpus * sizeof(uint64_t));
+        memset(current_event->prev_time_running, 0, number_of_cpus * sizeof(uint64_t));
+    }
+
+    for(group = 0; group < EV_GROUP_NUM; group++) {
+        group_leader_fds[group] = mallocz(number_of_cpus * sizeof(int));
+        memset(group_leader_fds[group], NO_FD, number_of_cpus * sizeof(int));
+    }
+
+    memset(&perf_event_attr, 0, sizeof(perf_event_attr));
+
+    for(cpu = 0; cpu < number_of_cpus; cpu++) {
+        for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+            if(unlikely(current_event->disabled)) continue;
+
+            perf_event_attr.type = current_event->type;
+            perf_event_attr.config = current_event->config;
+            perf_event_attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING;
+
+            int fd, group_leader_fd = *(*current_event->group_leader_fd + cpu);
+
+            fd = syscall(
+                __NR_perf_event_open,
+                &perf_event_attr,
+                ALL_PIDS,
+                cpu,
+                group_leader_fd,
+                flags
+            );
+
+            if(unlikely(group_leader_fd == NO_FD)) group_leader_fd = fd;
+
+            if(unlikely(fd < 0)) {
+                switch errno {
+                    case EACCES:
+                        error("Cannot access to the PMU: Permission denied");
+                        break;
+                    case EBUSY:
+                        error("Another event already has exclusive access to the PMU");
+                        break;
+                    default:
+                        error("Cannot open perf event");
+                }
+                error("Disabling event %u", current_event->id);
+                current_event->disabled = 1;
+            }
+
+            *(current_event->fd + cpu) = fd;
+            *(*current_event->group_leader_fd + cpu) = group_leader_fd;
+
+            if(unlikely(debug)) fprintf(stderr, "perf.plugin: event id = %u, cpu = %d, fd = %d, leader_fd = %d\n", current_event->id, cpu, fd, group_leader_fd);
+        }
+    }
+
+    return 0;
+}
+
+static void perf_free(void) {
+    int cpu, group;
+    struct perf_event *current_event = NULL;
+
+    for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+        for(cpu = 0; cpu < number_of_cpus; cpu++)
+            if(*(current_event->fd + cpu) != NO_FD) close(*(current_event->fd + cpu));
+
+        free(current_event->fd);
+        free(current_event->prev_value);
+        free(current_event->prev_time_enabled);
+        free(current_event->prev_time_running);
+    }
+
+    for(group = 0; group < EV_GROUP_NUM; group++)
+        free(group_leader_fds[group]);
+}
+
+static void reenable_events() {
+    int group, cpu;
+
+    for(group = 0; group < EV_GROUP_NUM; group++) {
+        for(cpu = 0; cpu < number_of_cpus; cpu++) {
+            int current_fd = *(group_leader_fds[group] + cpu);
+
+            if(unlikely(current_fd == NO_FD)) continue;
+
+            if(ioctl(current_fd, PERF_EVENT_IOC_DISABLE, PERF_IOC_FLAG_GROUP) == -1
+               || ioctl(current_fd, PERF_EVENT_IOC_ENABLE, PERF_IOC_FLAG_GROUP) == -1)
+            {
+                error("Cannot reenable event group");
+            }
+        }
+    }
+}
+
+static int perf_collect() {
+    int cpu;
+    struct perf_event *current_event = NULL;
+    static uint64_t prev_cpu_cycles_value = 0;
+    struct {
+        uint64_t value;
+        uint64_t time_enabled;
+        uint64_t time_running;
+    } read_result;
+
+    for(current_event = &perf_events[0]; current_event->id != EV_ID_END; current_event++) {
+        current_event->updated = 0;
+        current_event->value = 0;
+
+        if(unlikely(current_event->disabled)) continue;
+
+        for(cpu = 0; cpu < number_of_cpus; cpu++) {
+
+            ssize_t read_size = read(current_event->fd[cpu], &read_result, sizeof(read_result));
+
+            if(likely(read_size == sizeof(read_result))) {
+                if (likely(read_result.time_running
+                           && read_result.time_running != *(current_event->prev_time_running + cpu)
+                           && (read_result.time_enabled / read_result.time_running < RUNNING_THRESHOLD))) {
+                    current_event->value += (read_result.value - *(current_event->prev_value + cpu)) \
+                                             * (read_result.time_enabled - *(current_event->prev_time_enabled + cpu)) \
+                                             / (read_result.time_running - *(current_event->prev_time_running + cpu));
+                }
+
+                *(current_event->prev_value + cpu) = read_result.value;
+                *(current_event->prev_time_enabled + cpu) = read_result.time_enabled;
+                *(current_event->prev_time_running + cpu) = read_result.time_running;
+
+                current_event->updated = 1;
+            }
+            else {
+                error("Cannot update value for event %u", current_event->id);
+                return 1;
+            }
+        }
+
+        if(unlikely(debug)) fprintf(stderr, "perf.plugin: successfully read event id = %u, value = %lu\n", current_event->id, current_event->value);
+    }
+
+    if(unlikely(perf_events[EV_ID_CPU_CYCLES].value == prev_cpu_cycles_value))
+        reenable_events();
+    prev_cpu_cycles_value = perf_events[EV_ID_CPU_CYCLES].value;
+
+    return 0;
+}
+
+static void perf_send_metrics() {
+    static int // Hardware counters
+               cpu_cycles_chart_generated = 0,
+               instructions_chart_generated = 0,
+               branch_chart_generated = 0,
+               cache_chart_generated = 0,
+               bus_cycles_chart_generated = 0,
+               stalled_cycles_chart_generated = 0,
+
+               // Software counters
+               migrations_chart_generated = 0,
+               alighnment_chart_generated = 0,
+               emulation_chart_generated = 0,
+
+               // Hardware cache counters
+               L1D_chart_generated = 0,
+               L1D_prefetch_chart_generated = 0,
+               L1I_chart_generated = 0,
+               LL_chart_generated = 0,
+               DTLB_chart_generated = 0,
+               ITLB_chart_generated = 0,
+               PBU_chart_generated = 0;
+
+    // ------------------------------------------------------------------------
+
+    if(likely(perf_events[EV_ID_CPU_CYCLES].updated || perf_events[EV_ID_REF_CPU_CYCLES].updated)) {
+        if(unlikely(!cpu_cycles_chart_generated)) {
+            cpu_cycles_chart_generated = 1;
+
+            printf("CHART %s.%s '' 'CPU cycles' 'cycles/s' %s '' line %d %d %s\n"
+                   , RRD_TYPE_PERF
+                   , "cpu_cycles"
+                   , RRD_FAMILY_HW
+                   , NETDATA_CHART_PRIO_PERF_CPU_CYCLES
+                   , update_every
+                   , PLUGIN_PERF_NAME
+            );
+            printf("DIMENSION %s '' absolute 1 1\n", "cpu");
+            printf("DIMENSION %s '' absolute 1 1\n", "ref_cpu");
+        }
+
+        printf(
+               "BEGIN %s.%s\n"
+               , RRD_TYPE_PERF
+               , "cpu_cycles"
+        );
+        if(likely(perf_events[EV_ID_CPU_CYCLES].updated)) {
+            printf(
+                   "SET %s = %lld\n"
+                   , "cpu"
+                   , (collected_number) perf_events[EV_ID_CPU_CYCLES].value
+            );
+        }
+        if(likely(perf_events[EV_ID_REF_CPU_CYCLES].updated)) {
+            printf(
+                   "SET %s = %lld\n"
+                   , "ref_cpu"
+                   , (collected_number) perf_events[EV_ID_REF_CPU_CYCLES].value
+            );
+        }
+        printf("END\n");
+    }
+
+    // ------------------------------------------------------------------------
+
+    if(likely(perf_events[EV_ID_INSTRUCTIONS].updated)) {
+        if(unlikely(!instructions_chart_generated)) {
+            instructions_chart_generated = 1;
+
+            printf("CHART %s.%s '' 'Instructions' 'instructions/s' %s '' line %d %d %s\n"
+                   , RRD_TYPE_PERF
+                   , "instructions"
+                   , RRD_FAMILY_HW
+                   , NETDATA_CHART_PRIO_PERF_INSTRUCTIONS
+                   , update_every
+                   , PLUGIN_PERF_NAME
+            );
+            printf("DIMENSION %s '' absolute 1 1\n", "instructions");
+        }
+
+        printf(
+               "BEGIN %s.%s\n"
+               , RRD_TYPE_PERF
+               , "instructions"
+        );
+        printf(
+               "SET %s = %lld\n"
+               , "instructions"
+               , (collected_number) perf_events[EV_ID_INSTRUCTIONS].value
+        );
+        printf("END\n");
+    }
+
+    // ------------------------------------------------------------------------
+
+    if(likely(perf_events[EV_ID_BRANCH_INSTRUCTIONS].updated || perf_events[EV_ID_BRANCH_MISSES].updated)) {
+        if(unlikely(!branch_chart_generated)) {
+            branch_chart_generated = 1;
+
+            printf("CHART %s.%s '' 'Branch instructions' 'instructions/s' %s '' line %d %d %s\n"
+                   , RRD_TYPE_PERF
+                   , "branch_instructions"
+                   , RRD_FAMILY_HW
+                   , NETDATA_CHART_PRIO_PERF_BRANCH_INSTRUSTIONS
+                   , update_every
+                   , PLUGIN_PERF_NAME
+            );
+            printf("DIMENSION %s '' absolute 1 1\n", "instructions");
+            printf("DIMENSION %s '' absolute 1 1\n", "misses");
+        }
+
+        printf(
+               "BEGIN %s.%s\n"
+               , RRD_TYPE_PERF
+               , "branch_instructions"
+        );
+        if(likely(perf_events[EV_ID_BRANCH_INSTRUCTIONS].updated)) {
+            printf(
+                   "SET %s = %lld\n"
+                   , "instructions"
+                   , (collected_number) perf_events[EV_ID_BRANCH_INSTRUCTIONS].value
+            );
+        }
+        if(likely(perf_events[EV_ID_BRANCH_MISSES].updated)) {
+            printf(
+                   "SET %s = %lld\n"