Change source files structure for ZFS stats

6 files changed, 1005 insertions, 984 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b40704e040..74cab24ed6 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,6 +32,9 @@ set(NETDATA_SOURCE_FILES
         src/freebsd_getmntinfo.c
         src/freebsd_getifaddrs.c
         src/freebsd_devstat.c
+        src/zfs_common.c
+        src/zfs_common.h
+        src/freebsd_kstat_zfs.c
         src/global_statistics.c
         src/global_statistics.h
         src/health.c
diff --git a/src/Makefile.am b/src/Makefile.am
index 61a11f1e15..765a224a95 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -149,7 +149,9 @@ netdata_SOURCES += \
 	freebsd_getmntinfo.c \
 	freebsd_getifaddrs.c \
 	freebsd_devstat.c \
-	proc_spl_kstat_zfs.c \
+	zfs_common.c \
+	zfs_common.h \
+	freebsd_kstat_zfs.c \
 	$(NULL)
 else
 if MACOS
@@ -182,6 +184,8 @@ netdata_SOURCES += \
 	proc_net_softnet_stat.c \
 	proc_net_stat_conntrack.c \
 	proc_net_stat_synproxy.c \
+	zfs_common.c \
+    zfs_common.h \
 	proc_spl_kstat_zfs.c \
 	proc_stat.c \
 	proc_sys_kernel_random_entropy_avail.c \
diff --git a/src/freebsd_kstat_zfs.c b/src/freebsd_kstat_zfs.c
new file mode 100644
index 0000000000..9ee5d61e46
--- /dev/null
+++ b/src/freebsd_kstat_zfs.c
@@ -0,0 +1,210 @@
+#include "common.h"
+#include "zfs_common.h"
+
+// kstat.zfs.misc.arcstats
+int do_kstat_zfs_misc_arcstats(int update_every, usec_t dt) {
+    (void)dt;
+
+    unsigned long long l2_size;
+    size_t uint64_t_size = sizeof(uint64_t);
+    static struct mibs {
+        int hits[5];
+        int misses[5];
+        int demand_data_hits[5];
+        int demand_data_misses[5];
+        int demand_metadata_hits[5];
+        int demand_metadata_misses[5];
+        int prefetch_data_hits[5];
+        int prefetch_data_misses[5];
+        int prefetch_metadata_hits[5];
+        int prefetch_metadata_misses[5];
+        int mru_hits[5];
+        int mru_ghost_hits[5];
+        int mfu_hits[5];
+        int mfu_ghost_hits[5];
+        int deleted[5];
+        int mutex_miss[5];
+        int evict_skip[5];
+        int evict_not_enough[5];
+        int evict_l2_cached[5];
+        int evict_l2_eligible[5];
+        int evict_l2_ineligible[5];
+        int evict_l2_skip[5];
+        int hash_elements[5];
+        int hash_elements_max[5];
+        int hash_collisions[5];
+        int hash_chains[5];
+        int hash_chain_max[5];
+        int p[5];
+        int c[5];
+        int c_min[5];
+        int c_max[5];
+        int size[5];
+        int hdr_size[5];
+        int data_size[5];
+        int metadata_size[5];
+        int other_size[5];
+        int anon_size[5];
+        int anon_evictable_data[5];
+        int anon_evictable_metadata[5];
+        int mru_size[5];
+        int mru_evictable_data[5];
+        int mru_evictable_metadata[5];
+        int mru_ghost_size[5];
+        int mru_ghost_evictable_data[5];
+        int mru_ghost_evictable_metadata[5];
+        int mfu_size[5];
+        int mfu_evictable_data[5];
+        int mfu_evictable_metadata[5];
+        int mfu_ghost_size[5];
+        int mfu_ghost_evictable_data[5];
+        int mfu_ghost_evictable_metadata[5];
+        int l2_hits[5];
+        int l2_misses[5];
+        int l2_feeds[5];
+        int l2_rw_clash[5];
+        int l2_read_bytes[5];
+        int l2_write_bytes[5];
+        int l2_writes_sent[5];
+        int l2_writes_done[5];
+        int l2_writes_error[5];
+        int l2_writes_lock_retry[5];
+        int l2_evict_lock_retry[5];
+        int l2_evict_reading[5];
+        int l2_evict_l1cached[5];
+        int l2_free_on_write[5];
+        int l2_cdata_free_on_write[5];
+        int l2_abort_lowmem[5];
+        int l2_cksum_bad[5];
+        int l2_io_error[5];
+        int l2_size[5];
+        int l2_asize[5];
+        int l2_hdr_size[5];
+        int l2_compress_successes[5];
+        int l2_compress_zeros[5];
+        int l2_compress_failures[5];
+        int memory_throttle_count[5];
+        int duplicate_buffers[5];
+        int duplicate_buffers_size[5];
+        int duplicate_reads[5];
+        int memory_direct_count[5];
+        int memory_indirect_count[5];
+        int arc_no_grow[5];
+        int arc_tempreserve[5];
+        int arc_loaned_bytes[5];
+        int arc_prune[5];
+        int arc_meta_used[5];
+        int arc_meta_limit[5];
+        int arc_meta_max[5];
+        int arc_meta_min[5];
+        int arc_need_free[5];
+        int arc_sys_free[5];
+    } mibs;
+
+    l2exist = -1;
+
+    if(unlikely(sysctlbyname("kstat.zfs.misc.arcstats.l2_size", &l2_size, &uint64_t_size, NULL, 0)))
+        return 0;
+
+    if(likely(l2_size))
+        l2exist = 1;
+    else
+        l2exist = 0;
+
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hits", mibs.hits, arcstats.hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.misses", mibs.misses, arcstats.misses);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_hits", mibs.demand_data_hits, arcstats.demand_data_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_data_misses", mibs.demand_data_misses, arcstats.demand_data_misses);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_hits", mibs.demand_metadata_hits, arcstats.demand_metadata_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.demand_metadata_misses", mibs.demand_metadata_misses, arcstats.demand_metadata_misses);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_hits", mibs.prefetch_data_hits, arcstats.prefetch_data_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_data_misses", mibs.prefetch_data_misses, arcstats.prefetch_data_misses);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_hits", mibs.prefetch_metadata_hits, arcstats.prefetch_metadata_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.prefetch_metadata_misses", mibs.prefetch_metadata_misses, arcstats.prefetch_metadata_misses);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_hits", mibs.mru_hits, arcstats.mru_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_hits", mibs.mru_ghost_hits, arcstats.mru_ghost_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_hits", mibs.mfu_hits, arcstats.mfu_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_hits", mibs.mfu_ghost_hits, arcstats.mfu_ghost_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.deleted", mibs.deleted, arcstats.deleted);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mutex_miss", mibs.mutex_miss, arcstats.mutex_miss);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_skip", mibs.evict_skip, arcstats.evict_skip);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_not_enough", mibs.evict_not_enough, arcstats.evict_not_enough);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_cached", mibs.evict_l2_cached, arcstats.evict_l2_cached);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_eligible", mibs.evict_l2_eligible, arcstats.evict_l2_eligible);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_ineligible", mibs.evict_l2_ineligible, arcstats.evict_l2_ineligible);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.evict_l2_skip", mibs.evict_l2_skip, arcstats.evict_l2_skip);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements", mibs.hash_elements, arcstats.hash_elements);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_elements_max", mibs.hash_elements_max, arcstats.hash_elements_max);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_collisions", mibs.hash_collisions, arcstats.hash_collisions);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chains", mibs.hash_chains, arcstats.hash_chains);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hash_chain_max", mibs.hash_chain_max, arcstats.hash_chain_max);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.p", mibs.p, arcstats.p);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c", mibs.c, arcstats.c);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_min", mibs.c_min, arcstats.c_min);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.c_max", mibs.c_max, arcstats.c_max);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.size", mibs.size, arcstats.size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.hdr_size", mibs.hdr_size, arcstats.hdr_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.data_size", mibs.data_size, arcstats.data_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.metadata_size", mibs.metadata_size, arcstats.metadata_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.other_size", mibs.other_size, arcstats.other_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_size", mibs.anon_size, arcstats.anon_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_data", mibs.anon_evictable_data, arcstats.anon_evictable_data);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.anon_evictable_metadata", mibs.anon_evictable_metadata, arcstats.anon_evictable_metadata);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_size", mibs.mru_size, arcstats.mru_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_data", mibs.mru_evictable_data, arcstats.mru_evictable_data);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_evictable_metadata", mibs.mru_evictable_metadata, arcstats.mru_evictable_metadata);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_size", mibs.mru_ghost_size, arcstats.mru_ghost_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_data", mibs.mru_ghost_evictable_data, arcstats.mru_ghost_evictable_data);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mru_ghost_evictable_metadata", mibs.mru_ghost_evictable_metadata, arcstats.mru_ghost_evictable_metadata);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_size", mibs.mfu_size, arcstats.mfu_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_data", mibs.mfu_evictable_data, arcstats.mfu_evictable_data);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_evictable_metadata", mibs.mfu_evictable_metadata, arcstats.mfu_evictable_metadata);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_size", mibs.mfu_ghost_size, arcstats.mfu_ghost_size);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_data", mibs.mfu_ghost_evictable_data, arcstats.mfu_ghost_evictable_data);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.mfu_ghost_evictable_metadata", mibs.mfu_ghost_evictable_metadata, arcstats.mfu_ghost_evictable_metadata);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hits", mibs.l2_hits, arcstats.l2_hits);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_misses", mibs.l2_misses, arcstats.l2_misses);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_feeds", mibs.l2_feeds, arcstats.l2_feeds);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_rw_clash", mibs.l2_rw_clash, arcstats.l2_rw_clash);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_read_bytes", mibs.l2_read_bytes, arcstats.l2_read_bytes);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_write_bytes", mibs.l2_write_bytes, arcstats.l2_write_bytes);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_sent", mibs.l2_writes_sent, arcstats.l2_writes_sent);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_done", mibs.l2_writes_done, arcstats.l2_writes_done);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_error", mibs.l2_writes_error, arcstats.l2_writes_error);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_writes_lock_retry", mibs.l2_writes_lock_retry, arcstats.l2_writes_lock_retry);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_lock_retry", mibs.l2_evict_lock_retry, arcstats.l2_evict_lock_retry);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_reading", mibs.l2_evict_reading, arcstats.l2_evict_reading);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_evict_l1cached", mibs.l2_evict_l1cached, arcstats.l2_evict_l1cached);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_free_on_write", mibs.l2_free_on_write, arcstats.l2_free_on_write);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cdata_free_on_write", mibs.l2_cdata_free_on_write, arcstats.l2_cdata_free_on_write);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_abort_lowmem", mibs.l2_abort_lowmem, arcstats.l2_abort_lowmem);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_cksum_bad", mibs.l2_cksum_bad, arcstats.l2_cksum_bad);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_io_error", mibs.l2_io_error, arcstats.l2_io_error);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_size", mibs.l2_size, arcstats.l2_size);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_asize", mibs.l2_asize, arcstats.l2_asize);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_hdr_size", mibs.l2_hdr_size, arcstats.l2_hdr_size);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_successes", mibs.l2_compress_successes, arcstats.l2_compress_successes);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_zeros", mibs.l2_compress_zeros, arcstats.l2_compress_zeros);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.l2_compress_failures", mibs.l2_compress_failures, arcstats.l2_compress_failures);
+    GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_throttle_count", mibs.memory_throttle_count, arcstats.memory_throttle_count);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers", mibs.duplicate_buffers, arcstats.duplicate_buffers);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_buffers_size", mibs.duplicate_buffers_size, arcstats.duplicate_buffers_size);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.duplicate_reads", mibs.duplicate_reads, arcstats.duplicate_reads);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_direct_count", mibs.memory_direct_count, arcstats.memory_direct_count);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.memory_indirect_count", mibs.memory_indirect_count, arcstats.memory_indirect_count);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_no_grow", mibs.arc_no_grow, arcstats.arc_no_grow);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_tempreserve", mibs.arc_tempreserve, arcstats.arc_tempreserve);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_loaned_bytes", mibs.arc_loaned_bytes, arcstats.arc_loaned_bytes);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_prune", mibs.arc_prune, arcstats.arc_prune);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_used", mibs.arc_meta_used, arcstats.arc_meta_used);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_limit", mibs.arc_meta_limit, arcstats.arc_meta_limit);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_max", mibs.arc_meta_max, arcstats.arc_meta_max);
+    // not used: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_meta_min", mibs.arc_meta_min, arcstats.arc_meta_min);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_need_free", mibs.arc_need_free, arcstats.arc_need_free);
+    // missing mib: GETSYSCTL_SIMPLE("kstat.zfs.misc.arcstats.arc_sys_free", mibs.arc_sys_free, arcstats.arc_sys_free);
+
+    generate_charts_arcstats(update_every);
+    generate_charts_arc_summary(update_every);
+
+    return 0;
+}
\ No newline at end of file
diff --git a/src/proc_spl_kstat_zfs.c b/src/proc_spl_kstat_zfs.c
index 8cf359c163..77cf15dbcc 100644
--- a/src/proc_spl_kstat_zfs.c
+++ b/src/proc_spl_kstat_zfs.c
@@ -1,995 +1,16 @@
 #include "common.h"
+#include "zfs_common.h"
 
 #define ZFS_PROC_ARCSTATS "/proc/spl/kstat/zfs/arcstats"
-#define ZFS_FAMILY_SIZE "size"
-#define ZFS_FAMILY_EFFICIENCY "efficiency"
-#define ZFS_FAMILY_ACCESSES "accesses"
-#define ZFS_FAMILY_OPERATIONS "operations"
-#define ZFS_FAMILY_HASH "hashes"
 
-static struct arcstats {
-    unsigned long long hits;
-    unsigned long long misses;
-    unsigned long long demand_data_hits;
-    unsigned long long demand_data_misses;
-    unsigned long long demand_metadata_hits;
-    unsigned long long demand_metadata_misses;
-    unsigned long long prefetch_data_hits;
-    unsigned long long prefetch_data_misses;
-    unsigned long long prefetch_metadata_hits;
-    unsigned long long prefetch_metadata_misses;
-    unsigned long long mru_hits;
-    unsigned long long mru_ghost_hits;
-    unsigned long long mfu_hits;
-    unsigned long long mfu_ghost_hits;
-    unsigned long long deleted;
-    unsigned long long mutex_miss;
-    unsigned long long evict_skip;
-    unsigned long long evict_not_enough;
-    unsigned long long evict_l2_cached;
-    unsigned long long evict_l2_eligible;
-    unsigned long long evict_l2_ineligible;
-    unsigned long long evict_l2_skip;
-    unsigned long long hash_elements;
-    unsigned long long hash_elements_max;
-    unsigned long long hash_collisions;
-    unsigned long long hash_chains;
-    unsigned long long hash_chain_max;
-    unsigned long long p;
-    unsigned long long c;
-    unsigned long long c_min;
-    unsigned long long c_max;
-    unsigned long long size;
-    unsigned long long hdr_size;
-    unsigned long long data_size;
-    unsigned long long metadata_size;
-    unsigned long long other_size;
-    unsigned long long anon_size;
-    unsigned long long anon_evictable_data;
-    unsigned long long anon_evictable_metadata;
-    unsigned long long mru_size;
-    unsigned long long mru_evictable_data;
-    unsigned long long mru_evictable_metadata;
-    unsigned long long mru_ghost_size;
-    unsigned long long mru_ghost_evictable_data;
-    unsigned long long mru_ghost_evictable_metadata;
-    unsigned long long mfu_size;
-    unsigned long long mfu_evictable_data;
-    unsigned long long mfu_evictable_metadata;
-    unsigned long long mfu_ghost_size;
-    unsigned long long mfu_ghost_evictable_data;
-    unsigned long long mfu_ghost_evictable_metadata;
-    unsigned long long l2_hits;
-    unsigned long long l2_misses;
-    unsigned long long l2_feeds;
-    unsigned long long l2_rw_clash;
-    unsigned long long l2_read_bytes;
-    unsigned long long l2_write_bytes;
-    unsigned long long l2_writes_sent;
-    unsigned long long l2_writes_done;
-    unsigned long long l2_writes_error;
-    unsigned long long l2_writes_lock_retry;
-    unsigned long long l2_evict_lock_retry;
-    unsigned long long l2_evict_reading;
-    unsigned long long l2_evict_l1cached;
-    unsigned long long l2_free_on_write;
-    unsigned long long l2_cdata_free_on_write;
-    unsigned long long l2_abort_lowmem;
-    unsigned long long l2_cksum_bad;
-    unsigned long long l2_io_error;
-    unsigned long long l2_size;
-    unsigned long long l2_asize;
-    unsigned long long l2_hdr_size;
-    unsigned long long l2_compress_successes;
-    unsigned long long l2_compress_zeros;
-    unsigned long long l2_compress_failures;
-    unsigned long long memory_throttle_count;
-    unsigned long long duplicate_buffers;
-    unsigned long long duplicate_buffers_size;
-    unsigned long long duplicate_reads;
-    unsigned long long memory_direct_count;
-    unsigned long long memory_indirect_count;
-    unsigned long long arc_no_grow;
-    unsigned long long arc_tempreserve;
-    unsigned long long arc_loaned_bytes;
-    unsigned long long arc_prune;
-    unsigned long long arc_meta_used;
-    unsigned long long arc_meta_limit;
-    unsigned long long arc_meta_max;
-    unsigned long long arc_meta_min;
-    unsigned long long arc_need_free;
-    unsigned long long arc_sys_free;
-} arcstats = { 0 };
-
-int l2exist = -1;
-
-static void generate_charts_arcstats(int update_every) {
-
-    // ARC reads
-    unsigned long long aread = arcstats.hits + arcstats.misses;
-
-    // Demand reads
-    unsigned long long dhit = arcstats.demand_data_hits + arcstats.demand_metadata_hits;
-    unsigned long long dmiss = arcstats.demand_data_misses + arcstats.demand_metadata_misses;
-    unsigned long long dread = dhit + dmiss;
-
-    // Prefetch reads
-    unsigned long long phit = arcstats.prefetch_data_hits + arcstats.prefetch_metadata_hits;
-    unsigned long long pmiss = arcstats.prefetch_data_misses + arcstats.prefetch_metadata_misses;
-    unsigned long long pread = phit + pmiss;
-
-    // Metadata reads
-    unsigned long long mhit = arcstats.prefetch_metadata_hits + arcstats.demand_metadata_hits;
-    unsigned long long mmiss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses;
-    unsigned long long mread = mhit + mmiss;
-
-    // l2 reads
-    unsigned long long l2hit = arcstats.l2_hits + arcstats.l2_misses;
-    unsigned long long l2miss = arcstats.prefetch_metadata_misses + arcstats.demand_metadata_misses;
-    unsigned long long l2read = l2hit + l2miss;
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_arc_size = NULL;
-        static RRDDIM *rd_arc_size = NULL;
-        static RRDDIM *rd_arc_target_size = NULL;
-        static RRDDIM *rd_arc_target_min_size = NULL;
-        static RRDDIM *rd_arc_target_max_size = NULL;
-
-        if (unlikely(!st_arc_size)) {
-            st_arc_size = rrdset_create_localhost(
-                    "zfs"
-                    , "arc_size"
-                    , NULL
-                    , ZFS_FAMILY_SIZE
-                    , NULL
-                    , "ZFS ARC Size"
-                    , "MB"
-                    , 2000
-                    , update_every
-                    , RRDSET_TYPE_AREA
-            );
-
-            rd_arc_size            = rrddim_add(st_arc_size, "size",   "arcsz", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
-            rd_arc_target_size     = rrddim_add(st_arc_size, "target", NULL,    1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
-            rd_arc_target_min_size = rrddim_add(st_arc_size, "min",    "min (hard limit)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
-            rd_arc_target_max_size = rrddim_add(st_arc_size, "max",    "max (high water)", 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
-        }
-        else
-            rrdset_next(st_arc_size);
-
-        rrddim_set_by_pointer(st_arc_size, rd_arc_size,            arcstats.size);
-        rrddim_set_by_pointer(st_arc_size, rd_arc_target_size,     arcstats.c);
-        rrddim_set_by_pointer(st_arc_size, rd_arc_target_min_size, arcstats.c_min);
-        rrddim_set_by_pointer(st_arc_size, rd_arc_target_max_size, arcstats.c_max);
-        rrdset_done(st_arc_size);
-    }
-
-    // --------------------------------------------------------------------
-
-    if(likely(l2exist)) {
-        static RRDSET *st_l2_size = NULL;
-        static RRDDIM *rd_l2_size = NULL;
-        static RRDDIM *rd_l2_asize = NULL;
-
-        if (unlikely(!st_l2_size)) {
-            st_l2_size = rrdset_create_localhost(
-                    "zfs"
-                    , "l2_size"
-                    , NULL
-                    , ZFS_FAMILY_SIZE
-                    , NULL
-                    , "ZFS L2 ARC Size"
-                    , "MB"
-                    , 2000
-                    , update_every
-                    , RRDSET_TYPE_AREA
-            );
-
-            rd_l2_asize = rrddim_add(st_l2_size, "actual", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
-            rd_l2_size  = rrddim_add(st_l2_size, "size",   NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
-        }
-        else
-            rrdset_next(st_l2_size);
-
-        rrddim_set_by_pointer(st_l2_size, rd_l2_size,  arcstats.l2_size);
-        rrddim_set_by_pointer(st_l2_size, rd_l2_asize, arcstats.l2_asize);
-        rrdset_done(st_l2_size);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_reads = NULL;
-        static RRDDIM *rd_aread = NULL;
-        static RRDDIM *rd_dread = NULL;
-        static RRDDIM *rd_pread = NULL;
-        static RRDDIM *rd_mread = NULL;
-        static RRDDIM *rd_l2read = NULL;
-
-        if (unlikely(!st_reads)) {
-            st_reads = rrdset_create_localhost(
-                    "zfs"
-                    , "reads"
-                    , NULL
-                    , ZFS_FAMILY_ACCESSES
-                    , NULL
-                    , "ZFS Reads"
-                    , "reads/s"
-                    , 2010
-                    , update_every
-                    , RRDSET_TYPE_AREA
-            );
-
-            rd_aread  = rrddim_add(st_reads, "areads",  "arc",      1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_dread  = rrddim_add(st_reads, "dreads",  "demand",   1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_pread  = rrddim_add(st_reads, "preads",  "prefetch", 1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_mread  = rrddim_add(st_reads, "mreads",  "metadata", 1, 1, RRD_ALGORITHM_INCREMENTAL);
-
-            if(l2exist)
-                rd_l2read = rrddim_add(st_reads, "l2reads", "l2",       1, 1, RRD_ALGORITHM_INCREMENTAL);
-        }
-        else
-            rrdset_next(st_reads);
-
-        rrddim_set_by_pointer(st_reads, rd_aread,  aread);
-        rrddim_set_by_pointer(st_reads, rd_dread,  dread);
-        rrddim_set_by_pointer(st_reads, rd_pread,  pread);
-        rrddim_set_by_pointer(st_reads, rd_mread,  mread);
-
-        if(l2exist)
-            rrddim_set_by_pointer(st_reads, rd_l2read, l2read);
-
-        rrdset_done(st_reads);
-    }
-
-    // --------------------------------------------------------------------
-
-    if(likely(l2exist)) {
-        static RRDSET *st_l2bytes = NULL;
-        static RRDDIM *rd_l2_read_bytes = NULL;
-        static RRDDIM *rd_l2_write_bytes = NULL;
-
-        if (unlikely(!st_l2bytes)) {
-            st_l2bytes = rrdset_create_localhost(
-                    "zfs"
-                    , "bytes"
-                    , NULL
-                    , ZFS_FAMILY_ACCESSES
-                    , NULL
-                    , "ZFS ARC L2 Read/Write Rate"
-                    , "kilobytes/s"
-                    , 2200
-                    , update_every
-                    , RRDSET_TYPE_AREA
-            );
-
-            rd_l2_read_bytes  = rrddim_add(st_l2bytes, "read",  NULL,  1, 1024, RRD_ALGORITHM_INCREMENTAL);
-            rd_l2_write_bytes = rrddim_add(st_l2bytes, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
-        }
-        else
-            rrdset_next(st_l2bytes);
-
-        rrddim_set_by_pointer(st_l2bytes, rd_l2_read_bytes, arcstats.l2_read_bytes);
-        rrddim_set_by_pointer(st_l2bytes, rd_l2_write_bytes, arcstats.l2_write_bytes);
-        rrdset_done(st_l2bytes);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_ahits = NULL;
-        static RRDDIM *rd_ahits = NULL;
-        static RRDDIM *rd_amisses = NULL;
-
-        if (unlikely(!st_ahits)) {
-            st_ahits = rrdset_create_localhost(
-                    "zfs"
-                    , "hits"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS ARC Hits"
-                    , "percentage"
-                    , 2020
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-            rd_ahits   = rrddim_add(st_ahits, "hits", NULL,   1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-            rd_amisses = rrddim_add(st_ahits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-        }
-        else
-            rrdset_next(st_ahits);
-
-        rrddim_set_by_pointer(st_ahits, rd_ahits,   arcstats.hits);
-        rrddim_set_by_pointer(st_ahits, rd_amisses, arcstats.misses);
-        rrdset_done(st_ahits);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_dhits = NULL;
-        static RRDDIM *rd_dhits = NULL;
-        static RRDDIM *rd_dmisses = NULL;
-
-        if (unlikely(!st_dhits)) {
-            st_dhits = rrdset_create_localhost(
-                    "zfs"
-                    , "dhits"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS Demand Hits"
-                    , "percentage"
-                    , 2030
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-            rd_dhits   = rrddim_add(st_dhits, "hits",   NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-            rd_dmisses = rrddim_add(st_dhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-        }
-        else
-            rrdset_next(st_dhits);
-
-        rrddim_set_by_pointer(st_dhits, rd_dhits,   dhit);
-        rrddim_set_by_pointer(st_dhits, rd_dmisses, dmiss);
-        rrdset_done(st_dhits);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_phits = NULL;
-        static RRDDIM *rd_phits = NULL;
-        static RRDDIM *rd_pmisses = NULL;
-
-        if (unlikely(!st_phits)) {
-            st_phits = rrdset_create_localhost(
-                    "zfs"
-                    , "phits"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS Prefetch Hits"
-                    , "percentage"
-                    , 2040
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-            rd_phits   = rrddim_add(st_phits, "hits",   NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-            rd_pmisses = rrddim_add(st_phits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-        }
-        else
-            rrdset_next(st_phits);
-
-        rrddim_set_by_pointer(st_phits, rd_phits,   phit);
-        rrddim_set_by_pointer(st_phits, rd_pmisses, pmiss);
-        rrdset_done(st_phits);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_mhits = NULL;
-        static RRDDIM *rd_mhits = NULL;
-        static RRDDIM *rd_mmisses = NULL;
-
-        if (unlikely(!st_mhits)) {
-            st_mhits = rrdset_create_localhost(
-                    "zfs"
-                    , "mhits"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS Metadata Hits"
-                    , "percentage"
-                    , 2050
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-            rd_mhits   = rrddim_add(st_mhits, "hits",   NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-            rd_mmisses = rrddim_add(st_mhits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-        }
-        else
-            rrdset_next(st_mhits);
-
-        rrddim_set_by_pointer(st_mhits, rd_mhits,   mhit);
-        rrddim_set_by_pointer(st_mhits, rd_mmisses, mmiss);
-        rrdset_done(st_mhits);
-    }
-
-    // --------------------------------------------------------------------
-
-    if(likely(l2exist)) {
-        static RRDSET *st_l2hits = NULL;
-        static RRDDIM *rd_l2hits = NULL;
-        static RRDDIM *rd_l2misses = NULL;
-
-        if (unlikely(!st_l2hits)) {
-            st_l2hits = rrdset_create_localhost(
-                    "zfs"
-                    , "l2hits"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS L2 Hits"
-                    , "percentage"
-                    , 2060
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-            rd_l2hits   = rrddim_add(st_l2hits, "hits",   NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-            rd_l2misses = rrddim_add(st_l2hits, "misses", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL);
-        }
-        else
-            rrdset_next(st_l2hits);
-
-        rrddim_set_by_pointer(st_l2hits, rd_l2hits,   l2hit);
-        rrddim_set_by_pointer(st_l2hits, rd_l2misses, l2miss);
-        rrdset_done(st_l2hits);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_list_hits = NULL;
-        static RRDDIM *rd_mfu = NULL;
-        static RRDDIM *rd_mru = NULL;
-        static RRDDIM *rd_mfug = NULL;
-        static RRDDIM *rd_mrug = NULL;
-
-        if (unlikely(!st_list_hits)) {
-            st_list_hits = rrdset_create_localhost(
-                    "zfs"
-                    , "list_hits"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS List Hits"
-                    , "hits/s"
-                    , 2100
-                    , update_every
-                    , RRDSET_TYPE_AREA
-            );
-
-            rd_mfu = rrddim_add(st_list_hits,  "mfu",  NULL,        1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_mfug = rrddim_add(st_list_hits, "mfug", "mfu ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_mru = rrddim_add(st_list_hits,  "mru",  NULL,        1, 1, RRD_ALGORITHM_INCREMENTAL);
-            rd_mrug = rrddim_add(st_list_hits, "mrug", "mru ghost", 1, 1, RRD_ALGORITHM_INCREMENTAL);
-        }
-        else
-            rrdset_next(st_list_hits);
-
-        rrddim_set_by_pointer(st_list_hits, rd_mfu, arcstats.mfu_hits);
-        rrddim_set_by_pointer(st_list_hits, rd_mru, arcstats.mru_hits);
-        rrddim_set_by_pointer(st_list_hits, rd_mfug, arcstats.mfu_ghost_hits);
-        rrddim_set_by_pointer(st_list_hits, rd_mrug, arcstats.mru_ghost_hits);
-        rrdset_done(st_list_hits);
-    }
-}
-
-static void generate_charts_arc_summary(int update_every) {
-    unsigned long long arc_accesses_total = arcstats.hits + arcstats.misses;
-    unsigned long long real_hits = arcstats.mfu_hits + arcstats.mru_hits;
-    unsigned long long real_misses = arc_accesses_total - real_hits;
-
-    //unsigned long long anon_hits = arcstats.hits - (arcstats.mfu_hits + arcstats.mru_hits + arcstats.mfu_ghost_hits + arcstats.mru_ghost_hits);
-
-    unsigned long long arc_size = arcstats.size;
-    unsigned long long mru_size = arcstats.p;
-    //unsigned long long target_min_size = arcstats.c_min;
-    //unsigned long long target_max_size = arcstats.c_max;
-    unsigned long long target_size = arcstats.c;
-    //unsigned long long target_size_ratio = (target_max_size / target_min_size);
-
-    unsigned long long mfu_size;
-    if(arc_size > target_size)
-        mfu_size = arc_size - mru_size;
-    else
-        mfu_size = target_size - mru_size;
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_arc_size_breakdown = NULL;
-        static RRDDIM *rd_most_recent = NULL;
-        static RRDDIM *rd_most_frequent = NULL;
-
-        if (unlikely(!st_arc_size_breakdown)) {
-            st_arc_size_breakdown = rrdset_create_localhost(
-                    "zfs"
-                    , "arc_size_breakdown"
-                    , NULL
-                    , ZFS_FAMILY_EFFICIENCY
-                    , NULL
-                    , "ZFS ARC Size Breakdown"
-                    , "percentage"
-                    , 2020
-                    , update_every
-                    , RRDSET_TYPE_STACKED
-            );
-
-            rd_most_recent   = rrddim_add(st_arc_size_breakdown, "recent", NULL,   1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL);
-            rd_most_frequent = rrddim_add(st_arc_size_breakdown, "frequent", NULL, 1, 1, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL);
-        }
-        else
-            rrdset_next(st_arc_size_breakdown);
-
-        rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_recent,   mru_size);
-        rrddim_set_by_pointer(st_arc_size_breakdown, rd_most_frequent, mfu_size);
-        rrdset_done(st_arc_size_breakdown);
-    }
-
-    // --------------------------------------------------------------------
-
-    {
-        static RRDSET *st_memory = NULL;
-#ifndef __FreeBSD__
-        static RRDDIM *rd_direct = NULL;
-#endif
-        static RRDDIM *rd_throttled = NULL;
-#ifndef __FreeBSD__
-        static RRDDIM *rd_indirect = NULL;
-#endif
-
-        if (unlikely(!st_memory)) {