added RRDCALC management; preparation for expression evaluation

9 files changed, 586 insertions, 36 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e4bc52d46e..d5e6d195e3 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -82,7 +82,7 @@ set(NETDATA_SOURCE_FILES
         src/web_client.h
         src/web_server.c
         src/web_server.h
-        config.h src/health.h src/health.c)
+        config.h src/health.h src/health.c src/eval.h src/eval.c)
 
 set(APPS_PLUGIN_SOURCE_FILES
         src/appconfig.c
diff --git a/src/Makefile.am b/src/Makefile.am
index 2b698f8af3..8fa6d5bdf7 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -32,6 +32,7 @@ netdata_SOURCES = \
 	common.c common.h \
 	daemon.c daemon.h \
 	dictionary.c dictionary.h \
+	eval.c eval.h \
 	global_statistics.c global_statistics.h \
 	health.c health.h \
 	log.c log.h \
diff --git a/src/common.h b/src/common.h
index d614641e54..6344a69dbb 100644
--- a/src/common.h
+++ b/src/common.h
@@ -80,6 +80,7 @@
 #include "plugin_tc.h"
 #include "plugins_d.h"
 
+#include "eval.h"
 #include "health.h"
 
 #include "rrd.h"
diff --git a/src/eval.c b/src/eval.c
new file mode 100644
index 0000000000..d9377ecf6a
--- /dev/null
+++ b/src/eval.c
@@ -0,0 +1,289 @@
+#include "common.h"
+
+// ----------------------------------------------------------------------------
+// operators that work on 2 operands
+
+static inline int isoperatorterm_word(const char s) {
+    if(isspace(s) || s == '(' || s == '$' || s == '!' || s == '-' || s == '+' || isdigit(s)) return 1;
+    return 0;
+}
+
+static inline int isoperatorterm_symbol(const char s) {
+    if(isoperatorterm_word(s) || isalpha(s)) return 1;
+    return 0;
+}
+
+static inline int parse_and(const char **string) {
+    const char *s = *string;
+
+    // AND
+    if((s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && isoperatorterm_word(s[3])) {
+        *string = &s[4];
+        return 1;
+    }
+
+    // &&
+    if(s[0] == '&' && s[1] == '&' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_or(const char **string) {
+    const char *s = *string;
+
+    // OR
+    if((s[0] == 'O' || s[0] == '0') && (s[1] == 'R' || s[1] == 'r') && isoperatorterm_word(s[2])) {
+        *string = &s[3];
+        return 1;
+    }
+
+    // ||
+    if(s[0] == '|' && s[1] == '|' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_greater_than_or_equal(const char **string) {
+    const char *s = *string;
+
+    // >=
+    if(s[0] == '>' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_less_than_or_equal(const char **string) {
+    const char *s = *string;
+
+    // <=
+    if (s[0] == '<' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_greater(const char **string) {
+    const char *s = *string;
+
+    // >
+    if(s[0] == '>' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_less(const char **string) {
+    const char *s = *string;
+
+    // <
+    if(s[0] == '<' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_equal(const char **string) {
+    const char *s = *string;
+
+    // ==
+    if(s[0] == '=' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+        return 1;
+    }
+
+    // =
+    if(s[0] == '=' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_not_equal(const char **string) {
+    const char *s = *string;
+
+    // !=
+    if(s[0] == '!' && s[1] == '=' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+        return 1;
+    }
+
+    // <>
+    if(s[0] == '<' && s[1] == '>' && isoperatorterm_symbol(s[2])) {
+        *string = &s[2];
+    }
+
+    return 0;
+}
+
+static inline int parse_multiply(const char **string) {
+    const char *s = *string;
+
+    // *
+    if(s[0] == '*' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_divide(const char **string) {
+    const char *s = *string;
+
+    // /
+    if(s[0] == '/' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_minus(const char **string) {
+    const char *s = *string;
+
+    // -
+    if(s[0] == '-' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static inline int parse_plus(const char **string) {
+    const char *s = *string;
+
+    // +
+    if(s[0] == '+' && isoperatorterm_symbol(s[1])) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+// ----------------------------------------------------------------------------
+// operators that affect a single operand
+
+static inline int parse_not(const char **string) {
+    const char *s = *string;
+
+    // NOT
+    if((s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') && isoperatorterm_word(s[3])) {
+        *string = &s[4];
+        return 1;
+    }
+
+    if(s[0] == EVAL_OPERATOR_NOT) {
+        *string = &s[1];
+        return 1;
+    }
+
+    return 0;
+}
+
+static struct operator {
+    const char *printas;
+    int precedence;
+    char id;
+    int (*parse)(const char **);
+} operators[] = {
+        { "&&", 2, '&', parse_and },
+        { "||", 2, '|', parse_or },
+        { ">=", 3, '}', parse_greater_than_or_equal },
+        { "<=", 3, '{', parse_less_than_or_equal },
+        { "<>", 3, '~', parse_not_equal },
+        { "==", 3, '=', parse_equal },
+        { "<",  3, '<', parse_less },
+        { ">",  3, '>', parse_greater },
+        { "+",  4, EVAL_OPERATOR_PLUS, parse_plus },
+        { "-",  4, EVAL_OPERATOR_MINUS, parse_minus },
+        { "*",  5, '*', parse_multiply },
+        { "/",  5, '/', parse_divide },
+
+        // we should not put NOT in this list
+
+        { NULL, 0, EVAL_OPERATOR_NOP, NULL }
+};
+
+static inline char parse_operator(const char **s, int *precedence) {
+    int i;
+
+    for(i = 0 ; operators[i].parse != NULL ; i++)
+        if(operators[i].parse(s)) {
+            if(precedence) *precedence = operators[i].precedence;
+            return operators[i].id;
+        }
+
+    return EVAL_OPERATOR_NOP;
+}
+
+static inline EVAL_OPERAND *operand_alloc(int count) {
+    EVAL_OPERAND *op = calloc(1, sizeof(EVAL_OPERAND) + (sizeof(EVAL_VALUE) * count));
+    if(!op) fatal("Cannot allocate memory for OPERAND");
+
+    op->count = count;
+    return op;
+}
+
+static inline void operand_set_value_operand(EVAL_OPERAND *op, int pos, EVAL_OPERAND *value) {
+    if(pos >= op->count)
+        fatal("Invalid request to set position %d of OPERAND that has only %d values", pos + 1, op->count + 1);
+
+    op->ops[pos].type = EVAL_OPERAND_EXPRESSION;
+    op->ops[pos].expression = value;
+}
+
+// forward definitions
+static inline EVAL_OPERAND *parse_operand(const char **string);
+
+static inline EVAL_OPERAND *operand_alloc_single(const char **string, char type) {
+    EVAL_OPERAND *sub = parse_operand(string);
+    if(!sub) return NULL;
+
+    EVAL_OPERAND *op = operand_alloc(1);
+    if(!op) fatal("Cannot allocate memory for OPERAND");
+
+    op->operator = type;
+    operand_set_value_operand(op, 0, sub);
+    return op;
+}
+
+static inline EVAL_OPERAND *parse_operand(const char **string) {
+    const char *s = *string;
+    while(isspace(*s)) s++;
+
+    if(!*s) return NULL;
+    *string = s;
+
+    if(parse_not(string))
+        return operand_alloc_single(string, EVAL_OPERATOR_NOT);
+
+    if(parse_plus(string))
+        return operand_alloc_single(string, EVAL_OPERATOR_PLUS);
+
+    if(parse_minus(string))
+        return operand_alloc_single(string, EVAL_OPERATOR_MINUS);
+
+
+
+    return NULL;
+}
diff --git a/src/eval.h b/src/eval.h
new file mode 100644
index 0000000000..da3be13097
--- /dev/null
+++ b/src/eval.h
@@ -0,0 +1,38 @@
+#ifndef NETDATA_EVAL_H
+#define NETDATA_EVAL_H
+
+typedef struct variable {
+    char *name;
+    struct rrdvar *rrdvar;
+    struct variable *next;
+} VARIABLE;
+
+#define EVAL_OPERAND_INVALID 0
+#define EVAL_OPERAND_NUMBER 1
+#define EVAL_OPERAND_VARIABLE 2
+#define EVAL_OPERAND_EXPRESSION 3
+
+// these are used for EVAL_OPERAND.operator
+#define EVAL_OPERATOR_NOP   '\0'
+#define EVAL_OPERATOR_NOT   '!'
+#define EVAL_OPERATOR_PLUS  '+'
+#define EVAL_OPERATOR_MINUS '-'
+
+typedef struct eval_value {
+    int type;
+
+    union {
+        calculated_number number;
+        VARIABLE *variable;
+        struct eval_operand *expression;
+    };
+} EVAL_VALUE;
+
+typedef struct eval_operand {
+    char operator;
+
+    int count;
+    EVAL_VALUE ops[];
+} EVAL_OPERAND;
+
+#endif //NETDATA_EVAL_H
diff --git a/src/health.c b/src/health.c
index 92e8f87557..98e9013d3d 100644
--- a/src/health.c
+++ b/src/health.c
@@ -45,8 +45,15 @@ static inline RRDVAR *rrdvar_create(const char *name, uint32_t hash, int type, c
     return rv;
 }
 
-static inline void rrdvar_free(RRDVAR *rv) {
-    // FIXME: find all references of this and NULL them.
+static inline void rrdvar_free(RRDHOST *host, RRDVAR *rv) {
+    if(host) {
+        // FIXME: we may need some kind of locking here
+        // to have mutually exclusive access with eval()
+        VARIABLE *rf;
+        for (rf = host->references; rf; rf = rf->next)
+            if (rf->rrdvar == rv) rf->rrdvar = NULL;
+    }
+
     free(rv);
 }
 
@@ -59,7 +66,7 @@ static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *
         RRDVAR *ret = rrdvar_index_add(tree, rv);
         if(unlikely(ret != rv)) {
             debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", name, scope);
-            rrdvar_free(rv);
+            rrdvar_free(NULL, rv);
             rv = NULL;
         }
         else
@@ -147,12 +154,12 @@ void rrdsetvar_rename_all(RRDSET *st) {
             // name changed
             if (rs->context_name) {
                 rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_name);
-                rrdvar_free(rs->context_name);
+                rrdvar_free(st->rrdhost, rs->context_name);
             }
 
             if (rs->host_name) {
                 rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host_name);
-                rrdvar_free(rs->host_name);
+                rrdvar_free(st->rrdhost, rs->host_name);
             }
 
             free(rs->fullname);
@@ -163,6 +170,8 @@ void rrdsetvar_rename_all(RRDSET *st) {
             rs->host_name    = rrdvar_create_and_index("host",    &st->rrdhost->variables_root_index, rs->fullname, rs->hash_fullname, rs->type, rs->value);
         }
     }
+
+    rrdsetcalc_link_matching(st);
 }
 
 void rrdsetvar_free(RRDSETVAR *rs) {
@@ -171,27 +180,27 @@ void rrdsetvar_free(RRDSETVAR *rs) {
 
     if(rs->local) {
         rrdvar_index_del(&st->variables_root_index, rs->local);
-        rrdvar_free(rs->local);
+        rrdvar_free(st->rrdhost, rs->local);
     }
 
     if(rs->context) {
         rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context);
-        rrdvar_free(rs->context);
+        rrdvar_free(st->rrdhost, rs->context);
     }
 
     if(rs->host) {
         rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host);
-        rrdvar_free(rs->host);
+        rrdvar_free(st->rrdhost, rs->host);
     }
 
     if(rs->context_name) {
         rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_name);
-        rrdvar_free(rs->context_name);
+        rrdvar_free(st->rrdhost, rs->context_name);
     }
 
     if(rs->host_name) {
         rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host_name);
-        rrdvar_free(rs->host_name);
+        rrdvar_free(st->rrdhost, rs->host_name);
     }
 
     if(st->variables == rs) {
@@ -302,7 +311,7 @@ void rrddimvar_rename_all(RRDDIM *rd) {
             // name
             if (rs->local_name) {
                 rrdvar_index_del(&st->variables_root_index, rs->local_name);
-                rrdvar_free(rs->local_name);
+                rrdvar_free(st->rrdhost, rs->local_name);
             }
             free(rs->name);
             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
@@ -314,11 +323,11 @@ void rrddimvar_rename_all(RRDDIM *rd) {
             // fullidname
             if (rs->context_fullidname) {
                 rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullidname);
-                rrdvar_free(rs->context_fullidname);
+                rrdvar_free(st->rrdhost, rs->context_fullidname);
             }
             if (rs->host_fullidname) {
                 rrdvar_index_del(&st->rrdhost->variables_root_index, rs->context_fullidname);
-                rrdvar_free(rs->host_fullidname);
+                rrdvar_free(st->rrdhost, rs->host_fullidname);
             }
             free(rs->fullidname);
             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->name);
@@ -333,11 +342,11 @@ void rrddimvar_rename_all(RRDDIM *rd) {
             // fullnameid
             if (rs->context_fullnameid) {
                 rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullnameid);
-                rrdvar_free(rs->context_fullnameid);
+                rrdvar_free(st->rrdhost, rs->context_fullnameid);
             }
             if (rs->host_fullnameid) {
                 rrdvar_index_del(&st->rrdhost->variables_root_index, rs->context_fullnameid);
-                rrdvar_free(rs->host_fullnameid);
+                rrdvar_free(st->rrdhost, rs->host_fullnameid);
             }
             free(rs->fullnameid);
             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->id);
@@ -352,11 +361,11 @@ void rrddimvar_rename_all(RRDDIM *rd) {
             // fullnamename
             if (rs->context_fullnamename) {
                 rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullnamename);
-                rrdvar_free(rs->context_fullnamename);
+                rrdvar_free(st->rrdhost, rs->context_fullnamename);
             }
             if (rs->host_fullnamename) {
                 rrdvar_index_del(&st->rrdhost->variables_root_index, rs->context_fullnamename);
-                rrdvar_free(rs->host_fullnamename);
+                rrdvar_free(st->rrdhost, rs->host_fullnamename);
             }
             free(rs->fullnamename);
             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->name);
@@ -378,45 +387,45 @@ void rrddimvar_free(RRDDIMVAR *rs) {
 
     if(rs->local_id) {
         rrdvar_index_del(&st->variables_root_index, rs->local_id);
-        rrdvar_free(rs->local_id);
+        rrdvar_free(st->rrdhost, rs->local_id);
     }
     if(rs->local_name) {
         rrdvar_index_del(&st->variables_root_index, rs->local_name);
-        rrdvar_free(rs->local_name);
+        rrdvar_free(st->rrdhost, rs->local_name);
     }
 
     if(rs->context_fullidid) {
         rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullidid);
-        rrdvar_free(rs->context_fullidid);
+        rrdvar_free(st->rrdhost, rs->context_fullidid);
     }
     if(rs->context_fullidname) {
         rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullidname);
-        rrdvar_free(rs->context_fullidname);
+        rrdvar_free(st->rrdhost, rs->context_fullidname);
     }
     if(rs->context_fullnameid) {
         rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullnameid);
-        rrdvar_free(rs->context_fullnameid);
+        rrdvar_free(st->rrdhost, rs->context_fullnameid);
     }
     if(rs->context_fullnamename) {
         rrdvar_index_del(&st->rrdcontext->variables_root_index, rs->context_fullnamename);
-        rrdvar_free(rs->context_fullnamename);
+        rrdvar_free(st->rrdhost, rs->context_fullnamename);
     }
 
     if(rs->host_fullidid) {
         rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host_fullidid);
-        rrdvar_free(rs->host_fullidid);
+        rrdvar_free(st->rrdhost, rs->host_fullidid);
     }
     if(rs->host_fullidname) {
         rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host_fullidname);
-        rrdvar_free(rs->host_fullidname);
+        rrdvar_free(st->rrdhost, rs->host_fullidname);
     }
     if(rs->host_fullnameid) {
         rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host_fullnameid);
-        rrdvar_free(rs->host_fullnameid);
+        rrdvar_free(st->rrdhost, rs->host_fullnameid);
     }
     if(rs->host_fullnamename) {
         rrdvar_index_del(&st->rrdhost->variables_root_index, rs->host_fullnamename);
-        rrdvar_free(rs->host_fullnamename);
+        rrdvar_free(st->rrdhost, rs->host_fullnamename);
     }
 
     if(rd->variables == rs) {
@@ -441,3 +450,141 @@ void rrddimvar_free(RRDDIMVAR *rs) {
     free(rs->fullnamename);
     free(rs);
 }
+
+// ----------------------------------------------------------------------------
+// RRDCALC management
+
+// this has to be called while the caller has locked
+// the RRDHOST
+static inline void rrdhostcalc_linked(RRDHOST *host, RRDCALC *rc) {
+    // move it to be last
+
+    if(!rc->next)
+        // we are last already
+        return;
+
+    RRDCALC *t, *last = NULL, *prev = NULL;
+    for (t = host->calculations; t ; t = t->next) {
+        if(t->next == rc)
+            prev = t;
+
+        if(!t->next)
+            last = t;
+    }
+
+    if(!last) {
+        error("RRDCALC '%s' cannot be linked to the end of host '%s' list", rc->name, host->hostname);
+        return;
+    }
+
+    if(prev)
+        prev->next = rc->next;
+    else {
+        if(host->calculations == rc)
+            host->calculations = rc->next;
+        else {
+            error("RRDCALC '%s' is not found in host '%s' list", rc->name, host->hostname);
+            return;
+        }
+    }
+
+    last->next = rc;
+    rc->next = NULL;
+}
+
+// this has to be called while the caller has locked
+// the RRDHOST
+static inline void rrdhostcalc_unlinked(RRDHOST *host, RRDCALC *rc) {
+    // move it to be first
+
+    if(host->calculations == rc) {
+        // ok, we are the first
+        return;
+    }
+    else {
+        // find the previous one
+        RRDCALC *t;
+        for (t = host->calculations; t && t->next != rc; rc = rc->next) ;
+        if(unlikely(!t)) {
+            error("RRDCALC '%s' is not linked to host '%s'.", rc->name, host->hostname);
+            return;
+        }
+        t->next = rc->next;
+        rc->next = host->calculations;
+        host->calculations = rc;
+    }
+}
+
+static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
+    rc->rrdset = st;
+
+    rc->local   = rrdvar_create_and_index("local", &st->variables_root_index, rc->name, rc->hash, RRDVAR_TYPE_CALCULATED, &rc->value);
+    rc->context = rrdvar_create_and_index("context", &st->rrdcontext->variables_root_index, rc->name, rc->hash, RRDVAR_TYPE_CALCULATED, &rc->value);
+    rc->host    = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rc->name, rc->hash, RRDVAR_TYPE_CALCULATED, &rc->value);
+
+    rrdhostcalc_linked(st->rrdhost, rc);
+}
+
+// this has to be called while the RRDHOST is locked
+void rrdsetcalc_link_matching(RRDSET *st) {
+    RRDCALC *rc;
+
+    for(rc = st->rrdhost->calculations; rc ; rc = rc->next) {
+        // since unlinked ones are in front and linked at the end
+        // we stop on the first linked RRDCALC
+        if(rc->rrdset != NULL) break;
+
+        if((rc->hash_chart == st->hash && !strcmp(rc->name, st->id)) ||
+                (rc->hash_chart == st->hash_name && !strcmp(rc->name, st->name))) {
+            rrdsetcalc_link(st, rc);
+        }
+    }
+}
+
+// this has to be called while the RRDHOST is locked
+void rrdsetcalc_unlink(RRDCALC *rc) {
+    RRDSET *st = rc->rrdset;
+
+    if(!st) {
+        error("Requested to unlink RRDCALC '%s' which is not linked to any RRDSET", rc->name);
+        return;
+    }
+
+    RRDHOST *host = st->rrdhost;
+
+    // unlink it
+    if(rc->rrdset_prev)
+        rc->rrdset_prev->rrdset_next = rc->rrdset_next;
+
+    if(rc->rrdset_next)
+        rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
+
+    if(st->calculations == rc)
+        st->calculations = rc->rrdset_next;
+
+    rc->rrdset_prev = rc->rrdset_next = NULL;
+
+    if(rc->local) {
+        rrdvar_index_del(&st->variables_root_index, rc->local);
+        rrdvar_free(st->rrdhost, rc->local);
+        rc->local = NULL;
+    }
+
+    if(rc->context) {
+        rrdvar_index_del(&st->rrdcontext->variables_root_index, rc->context);
+        rc->context = NULL;
+    }
+
+    if(rc->host) {
+        rrdvar_index_del(&st->rrdhost->variables_root_index, rc->host);
+        rc->host = NULL;
+    }
+
+    rc->rrdset = NULL;
+
+    // RRDCALC will remain in RRDHOST
+    // so that if the matching chart is found in the future
+    // it will be applied automatically
+
+    rrdhostcalc_unlinked(host, rc);
+}
diff --git a/src/health.h b/src/health.h
index b29a2c5bfa..760cee4fe1 100644
--- a/src/health.h
+++ b/src/health.h
@@ -44,6 +44,10 @@ typedef struct rrdvar {
 } RRDVAR;
 
 // variables linked to charts
+// We link variables to point the values that are already
+// calculated / processed by the normal data collection process
+// This means, there will be no speed penalty for using
+// these variables
 typedef struct rrdsetvar {
     char *fullid;               // chart type.chart id.variable
     uint32_t hash_fullid;
@@ -72,6 +76,10 @@ typedef struct rrdsetvar {
 
 
 // variables linked to dimensions
+// We link variables to point the values that are already
+// calculated / processed by the normal data collection process
+// This means, there will be no speed penalty for using
+// these variables
 typedef struct rrddimvar {
     char *prefix;
     char *suffix;
@@ -117,25 +125,71 @@ typedef struct rrddimvar {
     struct rrddimvar *next;
 } RRDDIMVAR;
 
+// additional calculated variables
+// These aggregate time-series data at fixed intervals
+// (defined in their update_every member below)
+// These increase the overhead of netdata.
+//
+// These calculations are allocated and linked (->next)
+// to RRDHOST.
+// Then are also linked to RRDSET (of course only when the
+// chart is found, via ->rrdset_next and ->rrdset_prev).
+// This double-linked list is maintained sorted at all times
+// having as RRDSET->calculations the RRDCALC to be processed
+// next.
 typedef struct rrdcalc {
-    avl avl;
+    char *name;
+    uint32_t hash;
+
+    char *chart;        // the chart name
+    uint32_t hash_chart;
+
+    char *dimensions;   // the chart dimensions
 
     int group;          // grouping method: average, max, etc.
     int before;         // ending point in time-series
     int after;          // starting point in time-series
+    uint32_t options;   // calculation options
     int update_every;   // update frequency for the calculation
 
-    const char *name;
+    time_t last_updated;
+    time_t next_update;
+
     calculated_number value;
 
     RRDVAR *local;
     RRDVAR *context;
     RRDVAR *host;
 
+    struct rrdset *rrdset;
+    struct rrdcalc *rrdset_next;
+    struct rrdcalc *rrdset_prev;
+
     struct rrdcalc *next;
-    struct rrdcalc *prev;
 } RRDCALC;
 
+
+// RRDCALCTEMPLATE
+// these are to be applied to charts found dynamically
+// based on their context.
+typedef struct rrdcalctemplate {
+    char *name;
+    uint32_t hash_name;
+
+    char *context;
+    uint32_t hash_context;
+
+    char *dimensions;
+
+    int group;          // grouping method: average, max, etc.
+    int before;         // ending point in time-series
+    int after;          // starting point in time-series
+    uint32_t options;   // calculation options
+    int update_every;   // update frequency for the calculation
+
+    struct rrdcalctemplate *next;
+} RRDCALCTEMPLATE;
+
 #include "rrd.h"
 
 extern void rrdsetvar_rename_all(RRDSET *st);
@@ -146,4 +200,7 @@ extern void rrddimvar_rename_all(RRDDIM *rd);
 extern RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options);
 extern void rrddimvar_free(RRDDIMVAR *rs);
 
+extern void rrdsetcalc_link_matching(RRDSET *st);
+extern void rrdsetcalc_unlink(RRDCALC *rc);
+
 #endif //NETDATA_HEALTH_H
diff --git a/src/rrd.c b/src/rrd.c
index a55af06e8c..2b50f971c8 100644
--- a/src/rrd.c
+++ b/src/rrd.c
@@ -531,6 +531,8 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const
 
     rrdset_index_add(&localhost, st);
 
+    rrdsetcalc_link_matching(st);
+
 	pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
 
 	return(st);
@@ -740,6 +742,9 @@ void rrdset_free_all(void)
         while(st->variables)
             rrdsetvar_free(st->variables);
 
+        while(st->calculations)
+            rrdsetcalc_unlink(st->calculations);
+
         while(st->dimensions)
 			rrddim_free(st, st->dimensions);
 
diff --git a/src/rrd.h b/src/rrd.h
index 7a5b6674b6..e7bdc5a8f6 100644
--- a/src/rrd.h
+++ b/src/rrd.h
@@ -254,6 +254,13 @@ struct rrdset {
 	struct rrdset *next;							// linking of rrdsets
 
 	// ------------------------------------------------------------------------
+	// local variables
+
+	avl_tree_lock variables_root_index;
+	RRDSETVAR *variables;
+	RRDCALC *calculations;
+
+	// ------------------------------------------------------------------------
 	// members for checking the data when loading from disk
 
 	unsigned long memsize;							// how much mem we have allocated for this (without dimensions)
@@ -266,11 +273,6 @@ struct rrdset {
 	avl_tree_lock dimensions_index;						// the root of the dimensions index
 	RRDDIM *dimensions;								// the actual data for every dimension
 
-    // ------------------------------------------------------------------------
-    // local variables
-
-    avl_tree_lock variables_root_index;
-    RRDSETVAR *variables;
 };
 typedef struct rrdset RRDSET;
 
@@ -290,6 +292,16 @@ struct rrdhost {
 
     avl_tree_lock rrdcontext_root_index;
     avl_tree_lock variables_root_index;
+
+	// all RRDCALCs are primarily allocated and linked here
+	// RRDCALCs may be linked to charts at any point
+	// (charts may or may not exist when these are loaded)
+	RRDCALC *calculations;
+
+    // all variable references are linked here
+    // RRDVARs may be free'd, so every time this happens
+    // we need to find all their references and invalidate them
+    VARIABLE *references;
 };
 typedef struct rrdhost RRDHOST;