diff options
| -rw-r--r--[-rwxr-xr-x] | conf.d/apps_groups.conf | 16 | ||||
| -rwxr-xr-x | src/procfile.c | 6 | ||||
| -rwxr-xr-x | src/rrd.c | 213 | ||||
| -rwxr-xr-x | src/rrd.h | 22 | ||||
| -rwxr-xr-x | src/rrd2json.c | 49 | ||||
| -rwxr-xr-x | src/storage_number.c | 72 | ||||
| -rwxr-xr-x | src/storage_number.h | 23 | ||||
| -rwxr-xr-x | src/unit_test.c | 42 |
8 files changed, 312 insertions, 131 deletions
diff --git a/conf.d/apps_groups.conf b/conf.d/apps_groups.conf index c7257bc554..ba0a50923b 100755..100644 --- a/conf.d/apps_groups.conf +++ b/conf.d/apps_groups.conf @@ -20,9 +20,9 @@ # If a group_name starts with a -, the dimension will be hidden (cpu chart only) # -compile: cc1 cc1plus as gcc make automake autoconf +compile: cc1 cc1plus as gcc ld make automake autoconf rsync: rsync -media: mplayer vlc xine mediatomb omxplayer omxplayer.bin +media: mplayer vlc xine mediatomb omxplayer omxplayer.bin kodi kodi.bin xbmc xbmc.bin mediacenter eventlircd squid: squid squid2 squid3 apache: apache apache2 mysql: mysqld mysql @@ -37,9 +37,11 @@ lighttpd: lighttpd ftpd: proftpd in.tftpd samba: smbd nmbd winbindd nfs: rpcbind rpc.statd rpc.idmapd rpc.mountd nfsd4 nfsd4_callbacks nfsd nfsiod -ssh: ssh sshd -X: gdm X lightdm xdm gnome-session gconfd-2 gnome-terminal gnome-screensaver gnome-settings-daemon pulseaudio -named: named +ssh: ssh sshd scp +X: X lightdm xdm pulseaudio gkrellm +xfce: xfwm4 xfdesktop xfce4-appfinder Thunar xfsettingsd xfce4-panel +gnome: gnome-session gdm gconfd-2 gnome-terminal gnome-screensaver gnome-settings-daemon +named: named rncd clam: clamd freshclam cups: cupsd cups-browsed ntp: ntpq ntpd @@ -53,6 +55,8 @@ openvpn: openvpn cron: cron atd ha: corosync hs_logd ha_logd stonithd ipvs: ipvs_syncmaster ipvs_syncbackup -kernel: kthreadd +kernel: kthreadd kauditd lockd khelper kdevtmpfs khungtaskd rpciod fsnotify_mark kthrotld iscsi_eh deferwq netdata: netdata apps.plugin charts.d.plugin crsproxy: crsproxy +wifi: hostapd wpa_supplicant +system: systemd-journal systemd-udevd systemd-logind udisks-glue udisks-daemon udevd udevd connmand ipv6_addrconf dbus-daemon diff --git a/src/procfile.c b/src/procfile.c index 60233bcfd9..26e717b2a1 100755 --- a/src/procfile.c +++ b/src/procfile.c @@ -279,7 +279,7 @@ procfile *procfile_open(const char *filename, const char *separators) { procfile *ff = malloc(sizeof(procfile) + PROCFILE_INITIAL_BUFFER); if(!ff) { - error(PF_PREFIX ": Cannot allocate memory for file '%s'. Reason: %s", ff->filename, strerror(errno)); + error(PF_PREFIX ": Cannot allocate memory for file '%s'. Reason: %s", filename, strerror(errno)); close(fd); return NULL; } @@ -295,7 +295,7 @@ procfile *procfile_open(const char *filename, const char *separators) { ff->words = pfwords_new(); if(!ff->lines || !ff->words) { - error(PF_PREFIX ": Cannot initialize parser for file '%s'. Reason: %s", ff->filename, strerror(errno)); + error(PF_PREFIX ": Cannot initialize parser for file '%s'. Reason: %s", filename, strerror(errno)); procfile_close(ff); return NULL; } @@ -311,7 +311,7 @@ procfile *procfile_open(const char *filename, const char *separators) { const char *s = separators; while(*s) ff->separators[(int)*s++] = PF_CHAR_IS_SEPARATOR; - debug(D_PROCFILE, "File '%s' opened.", ff->filename); + debug(D_PROCFILE, "File '%s' opened.", filename); return ff; } @@ -177,13 +177,14 @@ void rrd_stats_reset(RRD_STATS *st) st->last_updated.tv_sec = 0; st->last_updated.tv_usec = 0; st->current_entry = 0; + st->counter = 0; st->counter_done = 0; RRD_DIMENSION *rd; for(rd = st->dimensions; rd ; rd = rd->next) { rd->last_collected_time.tv_sec = 0; rd->last_collected_time.tv_usec = 0; - rd->current_entry = 0; + bzero(rd->values, rd->entries * sizeof(storage_number)); } } @@ -217,14 +218,15 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, if(st) { if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) { errno = 0; - error("File %s does not have our version. Clearing it.", fullfilename); + info("Initializing file %s.", fullfilename); bzero(st, size); } else if(strcmp(st->id, fullid) != 0) { errno = 0; - error("File %s does not have our id. Unmapping it.", fullfilename); - munmap(st, size); - st = NULL; + error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid); + // munmap(st, size); + // st = NULL; + bzero(st, size); } else if(st->memsize != size || st->entries != entries) { errno = 0; @@ -297,6 +299,8 @@ RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, st->last_collected_time.tv_usec = 0; st->counter_done = 0; + st->gap_when_lost_iterations = config_get_number(st->id, "gap when lost iterations above", DEFAULT_GAP_INTERPOLATIONS); + pthread_rwlock_init(&st->rwlock, NULL); pthread_rwlock_wrlock(&root_rwlock); @@ -328,7 +332,7 @@ RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) { errno = 0; - error("File %s does not have our version. Clearing it.", fullfilename); + info("Initializing file %s.", fullfilename); bzero(rd, size); } else if(rd->memsize != size) { @@ -363,9 +367,10 @@ RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char } else if(strcmp(rd->id, id) != 0) { errno = 0; - error("File %s does not have our dimension id. Unmapping it.", fullfilename); - munmap(rd, size); - rd = NULL; + error("File %s contents are not for dimension %s. Clearing it.", fullfilename, id); + // munmap(rd, size); + // rd = NULL; + bzero(rd, size); } } @@ -597,6 +602,7 @@ void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collec gettimeofday(&rd->last_collected_time, NULL); rd->collected_value = value; + rd->updated = 1; } int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value) @@ -642,7 +648,7 @@ unsigned long long rrd_stats_done(RRD_STATS *st) debug(D_RRD_CALLS, "rrd_stats_done() for chart %s", st->name); RRD_DIMENSION *rd, *last; - int oldstate; + int oldstate, store_this_entry = 1; if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0) error("Cannot set pthread cancel state to DISABLE."); @@ -650,45 +656,81 @@ unsigned long long rrd_stats_done(RRD_STATS *st) // a read lock is OK here pthread_rwlock_rdlock(&st->rwlock); + // check if the chart has a long time to be refreshed if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) { - info("Chart chart %s took too long to be updated (%0.3Lf secs). Reseting chart history.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); + info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); rrd_stats_reset(st); st->usec_since_last_update = st->update_every * 1000000ULL; } - if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update); - if(!st->last_collected_time.tv_sec) gettimeofday(&st->last_collected_time, NULL); + // set last_collected_time + if(!st->last_collected_time.tv_sec) { + // it is the first entry + // set the last_collected_time to now + gettimeofday(&st->last_collected_time, NULL); + + // the first entry should not be stored + store_this_entry = 0; + + if(st->debug) debug(D_RRD_STATS, "%s: initializing last_collected to now. Will not store the next entry.", st->name); + } else { + // it is not the first entry + // calculate the proper last_collected_time, using usec_since_last_update unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update; st->last_collected_time.tv_sec = ut / 1000000ULL; st->last_collected_time.tv_usec = ut % 1000000ULL; } + // if this set has not been updated in the past + // we fake the last_update time to be = now - usec_since_last_update if(!st->last_updated.tv_sec) { + // it has never been updated before + // set a fake last_updated, in the past using usec_since_last_update unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; st->last_updated.tv_sec = ut / 1000000ULL; st->last_updated.tv_usec = ut % 1000000ULL; - if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0); + // the first entry should not be stored + store_this_entry = 0; + + if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0); } - if(usecdiff(&st->last_collected_time, &st->last_updated) > st->entries * st->update_every * 1000000ULL) { - info("History of chart %s too old (last updated at %u, last collected at %u). Reseting chart.", st->name, st->last_updated.tv_sec, st->last_collected_time.tv_sec); + // check if we will re-write the entire data set + if(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL) { + info("%s: too old data (last updated at %u.%u, last collected at %u.%u). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec); rrd_stats_reset(st); + st->usec_since_last_update = st->update_every * 1000000ULL; - pthread_rwlock_unlock(&st->rwlock); - return(st->usec_since_last_update); + + gettimeofday(&st->last_collected_time, NULL); + + unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update; + st->last_updated.tv_sec = ut / 1000000ULL; + st->last_updated.tv_usec = ut % 1000000ULL; + + // the first entry should not be stored + store_this_entry = 0; } + // these are the 3 variables that will help us in interpolation + // last_ut = the last time we added a value to the storage + // now_ut = the time the current value is taken at + // next_ut = the time of the next interpolation point unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec; - unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; + unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec; unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL; if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); if(st->debug) debug(D_RRD_STATS, "%s: now ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0); if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); + if(!st->counter_done) { + store_this_entry = 0; + if(st->debug) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name); + } st->counter_done++; // calculate totals and count the dimensions @@ -697,10 +739,13 @@ unsigned long long rrd_stats_done(RRD_STATS *st) for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ ) st->collected_total += rd->collected_value; + uint32_t storage_flags = SN_EXISTS; + // process all dimensions to calculate their values // based on the collected figures only // at this stage we do not interpolate anything for( rd = st->dimensions ; rd ; rd = rd->next ) { + if(st->debug) debug(D_RRD_STATS, "%s/%s: " " last_collected_value = " COLLECTED_NUMBER_FORMAT " collected_value = " COLLECTED_NUMBER_FORMAT @@ -760,7 +805,10 @@ unsigned long long rrd_stats_done(RRD_STATS *st) case RRD_DIMENSION_INCREMENTAL: // if the new is smaller than the old (an overflow, or reset), set the old equal to the new // to reset the calculation (it will give zero as the calculation for this second) - if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value; + if(rd->last_collected_value > rd->collected_value) { + storage_flags = SN_EXISTS_RESET; + rd->last_collected_value = rd->collected_value; + } rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value); @@ -801,39 +849,16 @@ unsigned long long rrd_stats_done(RRD_STATS *st) break; } } - // at this point we have all the calculated values ready - - if(st->counter_done == 1 || next_ut > now_ut) { - // we don't have any usable data yet - if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done); - - for( rd = st->dimensions; rd ; rd = rd->next ) { - rd->last_calculated_value = rd->calculated_value; - rd->last_collected_value = rd->collected_value; - switch(rd->algorithm) { - case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL: - case RRD_DIMENSION_INCREMENTAL: - if(!st->usec_since_last_update) rd->calculated_value = 0; - // keep the previous values - // the next time, a new incremental total will be calculated - break; - } - - rd->collected_value = 0; - } - st->last_collected_total = st->collected_total; + // at this point we have all the calculated values ready + // it is now time to interpolate values on a second boundary - pthread_rwlock_unlock(&st->rwlock); - if(pthread_setcancelstate(oldstate, NULL) != 0) - error("Cannot set pthread cancel state to RESTORE (%d).", oldstate); + unsigned long long first_ut = last_ut; + int iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL); - return(st->usec_since_last_update); - } + for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL, iterations-- ) { + if(iterations < 0) error("iterations calculation wrapped!"); - // it is now time to interpolate values on a second boundary - unsigned long long first_ut = last_ut; - for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL ) { if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0); if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0); @@ -894,28 +919,70 @@ unsigned long long rrd_stats_done(RRD_STATS *st) if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value; break; + } + if(!store_this_entry) { + store_this_entry = 1; + continue; + } + + if(rd->updated && iterations < st->gap_when_lost_iterations) { + rd->values[st->current_entry] = pack_storage_number( + new_value + * (calculated_number)rd->multiplier + / (calculated_number)rd->divisor + , storage_flags ); + + if(st->debug) + debug(D_RRD_STATS, "%s/%s: STORE[%ld] " + CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT + " * %ld" + " / %ld" + , st->id, rd->name + , st->current_entry + , unpack_storage_number(rd->values[st->current_entry]), new_value + , rd->multiplier + , rd->divisor + ); + } + else { + if(st->debug) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING " + , st->id, rd->name + , st->current_entry + ); + rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS); } + if(st->debug) { + calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor; + calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]); + calculated_number accuracy = accuracy_loss(t1, t2); + debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" + , st->id, rd->name + , st->current_entry + , t2 + , get_storage_number_flags(rd->values[st->current_entry]) + , t1 + , accuracy + , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" + ); + + rd->collected_volume += t1; + rd->stored_volume += t2; + accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume); + debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s" + , st->id, rd->name + , st->current_entry + , rd->stored_volume + , rd->collected_volume + , accuracy + , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : "" + ); - rd->values[st->current_entry] = pack_storage_number( - new_value - * (calculated_number)rd->multiplier - / (calculated_number)rd->divisor - ); - - if(st->debug) - debug(D_RRD_STATS, "%s/%s: STORE[%ld] " - CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT - " * %ld" - " / %ld" - , st->id, rd->name - , st->current_entry - , unpack_storage_number(rd->values[st->current_entry]), new_value - , rd->multiplier - , rd->divisor - ); + } } + // reset the storage flags for the next point, if any; + storage_flags = SN_EXISTS; if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) { // the db is overwriting values @@ -930,16 +997,28 @@ unsigned long long rrd_stats_done(RRD_STATS *st) } for( rd = st->dimensions; rd ; rd = rd->next ) { + if(!rd->updated) continue; rd->last_collected_value = rd->collected_value; rd->last_calculated_value = rd->calculated_value; rd->collected_value = 0; + rd->updated = 0; + + // if this is the first entry of incremental dimensions + // we have to set the first calculated_value to zero + // to eliminate the first spike + if(st->counter_done == 1) switch(rd->algorithm) { + case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL: + case RRD_DIMENSION_INCREMENTAL: + rd->calculated_value = 0; + // the next time, a new incremental total will be calculated + break; + } } st->last_collected_total = st->collected_total; // ALL DONE ABOUT THE DATA UPDATE // -------------------------------------------------------------------- - // find if there are any obsolete dimensions (not updated recently) for( rd = st->dimensions; rd ; rd = rd->next ) if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) @@ -14,14 +14,16 @@ extern int update_every; #define HISTORY_MAX (86400*10) extern int save_history; +#define DEFAULT_GAP_INTERPOLATIONS 10 + typedef long long total_number; #define TOTAL_NUMBER_FORMAT "%lld" #define RRD_STATS_NAME_MAX 1024 -#define RRD_STATS_MAGIC "NETDATA CACHE STATS FILE V009" -#define RRD_DIMENSION_MAGIC "NETDATA CACHE DIMENSION FILE V009" +#define RRD_STATS_MAGIC "NETDATA CACHE STATS FILE V010" +#define RRD_DIMENSION_MAGIC "NETDATA CACHE DIMENSION FILE V010" // ---------------------------------------------------------------------------- @@ -59,8 +61,8 @@ extern int memory_mode_id(const char *name); // ---------------------------------------------------------------------------- // algorithms types -#define RRD_DIMENSION_ABSOLUTE_NAME "absolute" -#define RRD_DIMENSION_INCREMENTAL_NAME "incremental" +#define RRD_DIMENSION_ABSOLUTE_NAME "absolute" +#define RRD_DIMENSION_INCREMENTAL_NAME "incremental" #define RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row" #define RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row" @@ -84,8 +86,8 @@ struct rrd_dimension { long entries; // how many entries this dimension has // this should be the same to the entries of the data set - long current_entry; // the entry that is currently being updated int update_every; // every how many seconds is this updated? + int updated; // set to 0 after each calculation, to 1 after each collected value int hidden; // if set to non zero, this dimension will not be sent to the client int mapped; // 1 if the file is mapped @@ -96,8 +98,8 @@ struct rrd_dimension { long divisor; struct timeval last_collected_time; // when was this dimension last updated - // this is only used to detect un-updated dimensions - // which are removed after some time + // this is actual date time we updated the last_collected_value + // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRD_STATS calculated_number calculated_value; calculated_number last_calculated_value; @@ -105,6 +107,9 @@ struct rrd_dimension { collected_number collected_value; // the value collected at this round collected_number last_collected_value; // the value that was collected at the last round + calculated_number collected_volume; + calculated_number stored_volume; + struct rrd_dimension *next; // linking of dimensions within the same data set storage_number values[]; // the array of values - THIS HAS TO BE THE LAST MEMBER @@ -135,6 +140,9 @@ struct rrd_stats { unsigned long hash; // a simple hash on the id, to speed up searching // we first compare hashes, and only if the hashes are equal we do string comparisons + int gap_when_lost_iterations; // after how many lost iterations a gap should be stored + // netdata will interpolate values for gaps lower than this + long priority; long entries; // total number of entries in the data set diff --git a/src/rrd2json.c b/src/rrd2json.c index c5659de207..cb00fce093 100755 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -264,12 +264,14 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int calculated_number print_values[dimensions]; // keep the final value to be printed int print_hidden[dimensions]; // keep hidden flags int found_non_zero[dimensions]; + int found_non_existing[dimensions]; // initialize them for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { group_values[c] = print_values[c] = 0; print_hidden[c] = rd->hidden; found_non_zero[c] = 0; + found_non_existing[c] = 0; } // ------------------------------------------------------------------------- @@ -348,6 +350,7 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int count++; group_count++; + // check if we have to print this now if(((count - pad) % group) == 0) { if(printed >= entries_to_show) { // debug(D_RRD_STATS, "Already printed all rows. Stopping."); @@ -356,8 +359,10 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int if(group_count != group) { // this is an incomplete group, skip it. - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { group_values[c] = 0; + found_non_existing[c] = 0; + } group_count = 0; continue; @@ -379,9 +384,17 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int print_this = 1; } + // do the calculations for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - calculated_number value = unpack_storage_number(rd->values[t]); + storage_number n = rd->values[t]; + calculated_number value = unpack_storage_number(n); + if(!does_storage_number_exist(n)) { + value = 0.0; + found_non_existing[c]++; + } + if(did_storage_number_reset(n)) annotate_reset = 1; + switch(group_method) { case GROUP_MAX: if(abs(value) > abs(group_values[c])) group_values[c] = value; @@ -394,19 +407,12 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int default: case GROUP_AVERAGE: group_values[c] += value; - if(print_this) group_values[c] /= group_count; + if(print_this) group_values[c] /= ( group_count - found_non_existing[c] ); break; } - - if(print_this) { - print_values[c] = group_values[c]; - group_values[c] = 0; - } } if(print_this) { - group_count = 0; - if(annotate_reset) { annotation_count++; web_buffer_strcpy(wb, overflow_annotation); @@ -417,22 +423,36 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int pc = 0; for(c = 0 ; c < dimensions ; c++) { - if(!print_hidden[c]) { + if(found_non_existing[c] == group_count) { + // all entries are non-existing pc++; web_buffer_strcpy(wb, pre_value); - web_buffer_rrd_value(wb, print_values[c]); + web_buffer_strcpy(wb, "null"); + web_buffer_strcpy(wb, post_value); + } + else if(!print_hidden[c]) { + pc++; + web_buffer_strcpy(wb, pre_value); + web_buffer_rrd_value(wb, group_values[c]); web_buffer_strcpy(wb, post_value); - if(print_values[c]) found_non_zero[c]++; + if(group_values[c]) found_non_zero[c]++; } + + // reset them for the next loop + group_values[c] = 0; + found_non_existing[c] = 0; } + + // if all dimensions are hidden, print a null if(!pc) { web_buffer_strcpy(wb, pre_value); - web_buffer_rrd_value(wb, (calculated_number)0); + web_buffer_strcpy(wb, "null"); web_buffer_strcpy(wb, post_value); } printed++; + group_count = 0; } } @@ -443,6 +463,7 @@ unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int int changed = 0; for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { group_values[c] = 0; + found_non_existing[c] = 0; if(!print_hidden[c] && !found_non_zero[c]) { changed = 1; diff --git a/src/storage_number.c b/src/storage_number.c index 97eea1c902..ccfff1e518 100755 --- a/src/storage_number.c +++ b/src/storage_number.c @@ -13,57 +13,70 @@ #endif #endif -storage_number pack_storage_number(calculated_number value) +storage_number pack_storage_number(calculated_number value, uint32_t flags) { - storage_number r = 0; - if(!value) return r; - - // bit 32 = sign + // bit 32 = sign 0:positive, 1:negative // bit 31 = 0:divide, 1:multiply - // bit 30, 29, 28 = (multiplier or divider) 0-7 - // bit 27 to 25 = reserved for flags + // bit 30, 29, 28 = (multiplier or divider) 0-6 (7 total) + // bit 27, 26, 25 flags // bit 24 to bit 1 = the value - storage_number sign = 0, exp = 0, mul; + storage_number r = get_storage_number_flags(flags); + if(!value) return r; + int m = 0; calculated_number n = value; + // if the value is negative + // add the sign bit and make it positive if(n < 0) { - sign = 1; + r += (1 << 31); // the sign bit 32 n = -n; } + // make its integer part fit in 0x00ffffff + // by dividing it by 10 up to 7 times + // and increasing the multiplier while(m < 7 && n > (calculated_number)0x00ffffff) { n /= 10; m++; } - while(m > -7 && n < (calculated_number)0x00199999) { - n *= 10; - m--; - } - if(m <= 0) { - exp = 0; - m = -m; - } - else exp = 1; + if(m) { + // the value was too big and we divided it + // so we add a multiplier to unpack it + r += (1 << 30) + (m << 27); // the multiplier m - if(n > (calculated_number)0x00ffffff) { - error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); - n = (calculated_number)0x00ffffff; + if(n > (calculated_number)0x00ffffff) { + error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); + r += 0x00ffffff; + return r; + } + } + else { + // 0x0019999e is the number that can be multiplied + // by 10 to give 0x00ffffff + // while the value is below 0x0019999e we can + // multiply it by 10, up to 7 times, increasing + // the multiplier + while(m < 7 && n < (calculated_number)0x0019999e) { + n *= 10; + m++; + } + + // the value was small enough and we multiplied it + // so we add a divider to unpack it + r += (0 << 30) + (m << 27); // the divider m } - - mul = m; #ifdef STORAGE_WITH_MATH // without this there are rounding problems // example: 0.9 becomes 0.89 - n = lrint(n); + r += lrint(n); +#else + r += (storage_number)n; #endif - r = (sign << 31) + (exp << 30) + (mul << 27) + n; - // fprintf(stderr, "PACK: %08X, sign = %d, exp = %d, mul = %d, n = " CALCULATED_NUMBER_FORMAT "\n", r, sign, exp, mul, n); - return r; } @@ -71,8 +84,9 @@ calculated_number unpack_storage_number(storage_number value) { if(!value) return 0; - int sign = 0; - int exp = 0; + int sign = 0, exp = 0; + + value ^= get_storage_number_flags(value); if(value & (1 << 31)) { sign = 1; diff --git a/src/storage_number.h b/src/storage_number.h index c21f427976..e26fde7876 100755 --- a/src/storage_number.h +++ b/src/storage_number.h @@ -15,7 +15,24 @@ typedef int32_t storage_number; typedef uint32_t ustorage_number; #define STORAGE_NUMBER_FORMAT "%d" -storage_number pack_storage_number(calculated_number value); +#define SN_NOT_EXISTS (0x0 << 24) +#define SN_EXISTS (0x1 << 24) +#define SN_EXISTS_RESET (0x2 << 24) +#define SN_EXISTS_UNDEF1 (0x3 << 24) +#define SN_EXISTS_UNDEF2 (0x4 << 24) +#define SN_EXISTS_UNDEF3 (0x5 << 24) +#define SN_EXISTS_UNDEF4 (0x6 << 24) + +#define SN_FLAGS_MASK (~(0x6 << 24)) + +// extract the flags +#define get_storage_number_flags(value) ((((storage_number)value) & (1 << 24)) | (((storage_number)value) & (2 << 24)) | (((storage_number)value) & (4 << 24))) + +// checks +#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0) +#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0) + +storage_number pack_storage_number(calculated_number value, uint32_t flags); calculated_number unpack_storage_number(storage_number value); int print_calculated_number(char *str, calculated_number value); @@ -25,4 +42,8 @@ int print_calculated_number(char *str, calculated_number value); #define STORAGE_NUMBER_NEGATIVE_MAX -0.00001 #define STORAGE_NUMBER_NEGATIVE_MIN -167772150000000.0 +// accepted accuracy loss +#define ACCURACY_LOSS 0.0001 +#define accuracy_loss(t1, t2) ((t1 == t2 || t1 == 0.0 || t2 == 0.0) ? 0.0 : (100.0 - ((t1 > t2) ? (t2 * 100.0 / t1 ) : (t1 * 100.0 / t2)))) + #endif /* NETDATA_STORAGE_NUMBER_H */ diff --git a/src/unit_test.c b/src/unit_test.c index d74cc95386..17edeec44f 100755 --- a/src/unit_test.c +++ b/src/unit_test.c @@ -8,14 +8,18 @@ #include "log.h" #include "web_buffer.h" -#define ACCURACY_LOSS 0.0000001 - int check_storage_number(calculated_number n, int debug) { char buffer[100]; + uint32_t flags = SN_EXISTS; - storage_number s = pack_storage_number(n); + storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); + if(!does_storage_number_exist(s)) { + fprintf(stderr, "Exists flags missing for number " CALCULATED_NUMBER_FORMAT "!\n", n); + return 5; + } + calculated_number ddiff = d - n; calculated_number dcdiff = ddiff * 100.0 / n; @@ -152,7 +156,7 @@ void benchmark_storage_number(int loop, int multiplier) { n *= multiplier; if(n > STORAGE_NUMBER_POSITIVE_MAX) n = STORAGE_NUMBER_POSITIVE_MIN; - s = pack_storage_number(n); + s = pack_storage_number(n, 1); d = unpack_storage_number(s); print_calculated_number(buffer, d); } @@ -177,8 +181,38 @@ void benchmark_storage_number(int loop, int multiplier) { } +static int check_storage_number_exists() { + uint32_t flags = SN_EXISTS; + + + for(flags = 0; flags < 7 ; flags++) { + if(get_storage_number_flags(flags << 24) != flags << 24) { + fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24)); + return 1; + } + } + + flags = SN_EXISTS; + calculated_number n = 0.0; + + storage_number s = pack_storage_number(n, flags); + calculated_number d = unpack_storage_number(s); + if(get_storage_number_flags(s) != flags) { + fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s)); + return 1; + } + if(n != d) { + fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d); + return 1; + } + + return 0; +} + int unit_test_storage() { + if(check_storage_number_exists()) return 0; + calculated_number c, a = 0; int i, j, g, r = 0; |