diff options
-rw-r--r-- | daemon/unit_test.c | 33 | ||||
-rw-r--r-- | database/engine/rrdengine.c | 2 | ||||
-rw-r--r-- | database/rrddim.c | 2 | ||||
-rw-r--r-- | database/rrdset.c | 28 | ||||
-rw-r--r-- | exporting/tests/test_exporting_engine.c | 20 | ||||
-rw-r--r-- | libnetdata/storage_number/storage_number.c | 18 | ||||
-rw-r--r-- | libnetdata/storage_number/storage_number.h | 17 | ||||
-rw-r--r-- | libnetdata/storage_number/tests/test_storage_number.c | 2 |
8 files changed, 65 insertions, 57 deletions
diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 2af9d45186..05b057b024 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -97,7 +97,7 @@ static int check_rrdcalc_comparisons(void) { int check_storage_number(calculated_number n, int debug) { char buffer[100]; - uint32_t flags = SN_EXISTS; + uint32_t flags = SN_DEFAULT_FLAGS; storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); @@ -150,7 +150,7 @@ calculated_number storage_number_min(calculated_number n) { do { last = n; n /= 2.0; - storage_number t = pack_storage_number(n, SN_EXISTS); + storage_number t = pack_storage_number(n, SN_DEFAULT_FLAGS); r = unpack_storage_number(t); } while(r != 0.0 && r != last); @@ -263,7 +263,7 @@ void benchmark_storage_number(int loop, int multiplier) { n *= multiplier; if(n > storage_number_positive_max) n = storage_number_positive_min; - s = pack_storage_number(n, SN_EXISTS); + s = pack_storage_number(n, SN_DEFAULT_FLAGS); d = unpack_storage_number(s); print_calculated_number(buffer, d); } @@ -289,25 +289,12 @@ void benchmark_storage_number(int loop, int multiplier) { } static int check_storage_number_exists() { - uint32_t flags; - - - for(flags = 0; flags < 7 ; flags++) { - if(get_storage_number_flags(flags << 24) != flags << 24) { - fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24)); - return 1; - } - } - - flags = SN_EXISTS; + uint32_t flags = SN_DEFAULT_FLAGS; calculated_number n = 0.0; storage_number s = pack_storage_number(n, flags); calculated_number d = unpack_storage_number(s); - if(get_storage_number_flags(s) != flags) { - fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s)); - return 1; - } + if(n != d) { fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d); return 1; @@ -1192,7 +1179,7 @@ int run_test(struct test *test) unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries; for(c = 0 ; c < max ; c++) { calculated_number v = unpack_storage_number(rd->values[c]); - calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_EXISTS)); + calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS)); int same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0; fprintf(stderr, " %s/%s: checking position %lu (at %lu secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n", test->name, rd->name, c+1, @@ -1667,7 +1654,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI for (k = 0; k < QUERY_BATCH; ++k) { last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c + k; - expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); n = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved); value = unpack_storage_number(n); @@ -1725,7 +1712,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] assert(rd[i][j] == d); last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c; - expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { @@ -1847,7 +1834,7 @@ int test_dbengine(void) assert(rd[i][j] == d); collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset; - calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS)); + calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS)); uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0; if(!same) { @@ -2076,7 +2063,7 @@ static void query_dbengine_chart(void *arg) ++thread_info->queries_nr; for (time_now = time_after ; time_now <= time_before ; time_now += update_every) { generatedv = generate_dbengine_chart_value(i, j, time_now); - expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_EXISTS)); + expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_DEFAULT_FLAGS)); if (unlikely(rd->state->query_ops.is_finished(&handle))) { if (!thread_info->delete_old_data) { /* data validation only when we don't delete */ diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c index f48181d29a..54a9cdf8dc 100644 --- a/database/engine/rrdengine.c +++ b/database/engine/rrdengine.c @@ -336,7 +336,7 @@ after_crc_check: /* care, we don't hold the descriptor mutex */ if (have_read_error) { /* Applications should make sure NULL values match 0 as does SN_EMPTY_SLOT */ - memset(page, 0, descr->page_length); + memset(page, SN_EMPTY_SLOT, descr->page_length); } else if (RRD_NO_COMPRESSION == header->compression_algorithm) { (void) memcpy(page, xt_io_descr->buf + payload_offset + page_offset, descr->page_length); } else { diff --git a/database/rrddim.c b/database/rrddim.c index 51c4428c72..5479ade7aa 100644 --- a/database/rrddim.c +++ b/database/rrddim.c @@ -119,7 +119,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) // RRDDIM legacy data collection functions static void rrddim_collect_init(RRDDIM *rd) { - rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS); + rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT; } static void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storage_number number) { (void)point_in_time; diff --git a/database/rrdset.c b/database/rrdset.c index e4701e3b9e..282b0e5ba1 100644 --- a/database/rrdset.c +++ b/database/rrdset.c @@ -1120,7 +1120,7 @@ static inline size_t rrdset_done_interpolate( , usec_t last_collect_ut , usec_t now_collect_ut , char store_this_entry - , uint32_t storage_flags + , uint32_t has_reset_value ) { RRDDIM *rd; @@ -1135,6 +1135,11 @@ static inline size_t rrdset_done_interpolate( size_t counter = st->counter; long current_entry = st->current_entry; + uint32_t storage_flags = SN_DEFAULT_FLAGS; + + if (has_reset_value) + storage_flags |= SN_EXISTS_RESET; + for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) { #ifdef NETDATA_INTERNAL_CHECKS @@ -1232,8 +1237,8 @@ static inline size_t rrdset_done_interpolate( } if(unlikely(!store_this_entry)) { - rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); //pack_storage_number(0, SN_NOT_EXISTS) -// rd->values[current_entry] = SN_EMPTY_SLOT; //pack_storage_number(0, SN_NOT_EXISTS); + rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); +// rd->values[current_entry] = SN_EMPTY_SLOT; continue; } @@ -1261,8 +1266,8 @@ static inline size_t rrdset_done_interpolate( ); #endif -// rd->values[current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS); - rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); //pack_storage_number(0, SN_NOT_EXISTS) +// rd->values[current_entry] = SN_EMPTY_SLOT; + rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); rd->last_stored_value = NAN; } @@ -1274,11 +1279,10 @@ static inline size_t rrdset_done_interpolate( calculated_number t2 = unpack_storage_number(rd->values[current_entry]); calculated_number accuracy = accuracy_loss(t1, t2); - debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" + debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)" , st->id, rd->name , current_entry , t2 - , get_storage_number_flags(rd->values[current_entry]) , t1 , accuracy , (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : "" @@ -1300,7 +1304,7 @@ static inline size_t rrdset_done_interpolate( #endif } // reset the storage flags for the next point, if any; - storage_flags = SN_EXISTS; + storage_flags = SN_DEFAULT_FLAGS; st->counter = ++counter; st->current_entry = current_entry = ((current_entry + 1) >= st->entries) ? 0 : current_entry + 1; @@ -1540,7 +1544,7 @@ after_first_database_work: st->collected_total += rd->collected_value; } - uint32_t storage_flags = SN_EXISTS; + uint32_t has_reset_value = 0; // process all dimensions to calculate their values // based on the collected figures only @@ -1637,7 +1641,7 @@ after_first_database_work: , rd->collected_value); if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS))) - storage_flags = SN_EXISTS_RESET; + has_reset_value = 1; uint64_t last = (uint64_t)rd->last_collected_value; uint64_t new = (uint64_t)rd->collected_value; @@ -1708,7 +1712,7 @@ after_first_database_work: ); if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS))) - storage_flags = SN_EXISTS_RESET; + has_reset_value = 1; rd->last_collected_value = rd->collected_value; } @@ -1787,7 +1791,7 @@ after_first_database_work: , last_collect_ut , now_collect_ut , store_this_entry - , storage_flags + , has_reset_value ); after_second_database_work: diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c index 845c96c3c9..7188c6eeeb 100644 --- a/exporting/tests/test_exporting_engine.c +++ b/exporting/tests/test_exporting_engine.c @@ -312,12 +312,12 @@ static void test_exporting_calculate_value_from_stored_data(void **state) expect_function_call(__mock_rrddim_query_is_finished); will_return(__mock_rrddim_query_is_finished, 0); expect_function_call(__mock_rrddim_query_next_metric); - will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_EXISTS)); + will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_DEFAULT_FLAGS)); expect_function_call(__mock_rrddim_query_is_finished); will_return(__mock_rrddim_query_is_finished, 0); expect_function_call(__mock_rrddim_query_next_metric); - will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_EXISTS)); + will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_DEFAULT_FLAGS)); expect_function_call(__mock_rrddim_query_is_finished); will_return(__mock_rrddim_query_is_finished, 1); @@ -431,7 +431,7 @@ static void test_format_dimension_stored_graphite_plaintext(void **state) struct engine *engine = *state; expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); RRDDIM *rd = localhost->rrdset_root->dimensions; assert_int_equal(format_dimension_stored_graphite_plaintext(engine->instance_root, rd), 0); @@ -459,7 +459,7 @@ static void test_format_dimension_stored_json_plaintext(void **state) struct engine *engine = *state; expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); RRDDIM *rd = localhost->rrdset_root->dimensions; assert_int_equal(format_dimension_stored_json_plaintext(engine->instance_root, rd), 0); @@ -487,7 +487,7 @@ static void test_format_dimension_stored_opentsdb_telnet(void **state) struct engine *engine = *state; expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); RRDDIM *rd = localhost->rrdset_root->dimensions; assert_int_equal(format_dimension_stored_opentsdb_telnet(engine->instance_root, rd), 0); @@ -515,7 +515,7 @@ static void test_format_dimension_stored_opentsdb_http(void **state) struct engine *engine = *state; expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); RRDDIM *rd = localhost->rrdset_root->dimensions; assert_int_equal(format_dimension_stored_opentsdb_http(engine->instance_root, rd), 0); @@ -1070,7 +1070,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) will_return(__wrap_now_realtime_sec, 2); expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(localhost, buffer, "test_server", "test_prefix", 0, 0); @@ -1087,7 +1087,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) will_return(__wrap_now_realtime_sec, 2); expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); rrd_stats_api_v1_charts_allmetrics_prometheus_single_host( localhost, buffer, "test_server", "test_prefix", 0, PROMETHEUS_OUTPUT_NAMES | PROMETHEUS_OUTPUT_TYPES); @@ -1106,7 +1106,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state) will_return(__wrap_now_realtime_sec, 2); expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(localhost, buffer, "test_server", "test_prefix", 0, 0); @@ -1265,7 +1265,7 @@ static void test_format_dimension_prometheus_remote_write(void **state) RRDDIM *rd = localhost->rrdset_root->dimensions; expect_function_call(__wrap_exporting_calculate_value_from_stored_data); - will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS)); + will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS)); expect_function_call(__wrap_add_metric); expect_value(__wrap_add_metric, write_request_p, 0xff); diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c index 8ef1353b09..3e6a9f45c4 100644 --- a/libnetdata/storage_number/storage_number.c +++ b/libnetdata/storage_number/storage_number.c @@ -2,17 +2,23 @@ #include "../libnetdata.h" +#define get_storage_number_flags(value) \ + ((((storage_number)(value)) & (1 << 24)) | \ + (((storage_number)(value)) & (1 << 25)) | \ + (((storage_number)(value)) & (1 << 26))) + storage_number pack_storage_number(calculated_number value, uint32_t flags) { // bit 32 = sign 0:positive, 1:negative // bit 31 = 0:divide, 1:multiply // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) // bit 27 SN_EXISTS_100 // bit 26 SN_EXISTS_RESET - // bit 25 SN_EXISTS + // bit 25 SN_ANOMALY_BIT = 0: anomalous, 1: not anomalous // bit 24 to bit 1 = the value storage_number r = get_storage_number_flags(flags); - if(!value) return r; + if(!value) + goto RET_SN; int m = 0; calculated_number n = value, factor = 10; @@ -47,7 +53,7 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) { error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value); #endif r += 0x00ffffff; - return r; + goto RET_SN; } } else { @@ -78,6 +84,10 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) { r += (storage_number)n; #endif +RET_SN: + if (r == SN_EMPTY_SLOT) + r = SN_ANOMALOUS_ZERO; + return r; } @@ -100,7 +110,7 @@ calculated_number unpack_storage_number(storage_number value) { factor = 100; // bit 26 SN_EXISTS_RESET - // bit 25 SN_EXISTS + // bit 25 SN_ANOMALY_BIT // bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total) int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27; diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h index 4ad7ff6246..4101f69e01 100644 --- a/libnetdata/storage_number/storage_number.h +++ b/libnetdata/storage_number/storage_number.h @@ -60,17 +60,24 @@ typedef long double collected_number; typedef uint32_t storage_number; #define STORAGE_NUMBER_FORMAT "%u" -#define SN_EXISTS (1 << 24) // the value exists +#define SN_ANOMALY_BIT (1 << 24) // the anomaly bit of the value #define SN_EXISTS_RESET (1 << 25) // the value has been overflown #define SN_EXISTS_100 (1 << 26) // very large value (multiplier is 100 instead of 10) -// extract the flags -#define get_storage_number_flags(value) ((((storage_number)(value)) & (1 << 24)) | (((storage_number)(value)) & (1 << 25)) | (((storage_number)(value)) & (1 << 26))) +#define SN_DEFAULT_FLAGS SN_ANOMALY_BIT + #define SN_EMPTY_SLOT 0x00000000 +// When the calculated number is zero and the value is anomalous (ie. it's bit +// is zero) we want to return a storage_number representation that is +// different from the empty slot. We achieve this by mapping zero to +// SN_EXISTS_100. Unpacking the SN_EXISTS_100 value will return zero because +// its fraction field (as well as its exponent factor field) will be zero. +#define SN_ANOMALOUS_ZERO SN_EXISTS_100 + // checks -#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0) -#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0) +#define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT) +#define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0) storage_number pack_storage_number(calculated_number value, uint32_t flags); calculated_number unpack_storage_number(storage_number value); diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/libnetdata/storage_number/tests/test_storage_number.c index 7ef18b1de9..f90521cabe 100644 --- a/libnetdata/storage_number/tests/test_storage_number.c +++ b/libnetdata/storage_number/tests/test_storage_number.c @@ -38,7 +38,7 @@ static void test_number_printing(void **state) print_calculated_number(value, -9999.9999999); assert_string_equal(value, "-9999.9999999"); - print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_EXISTS))); + print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS))); assert_string_equal(value, "16.77722"); } |