summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--daemon/unit_test.c33
-rw-r--r--database/engine/rrdengine.c2
-rw-r--r--database/rrddim.c2
-rw-r--r--database/rrdset.c28
-rw-r--r--exporting/tests/test_exporting_engine.c20
-rw-r--r--libnetdata/storage_number/storage_number.c18
-rw-r--r--libnetdata/storage_number/storage_number.h17
-rw-r--r--libnetdata/storage_number/tests/test_storage_number.c2
8 files changed, 65 insertions, 57 deletions
diff --git a/daemon/unit_test.c b/daemon/unit_test.c
index 2af9d45186..05b057b024 100644
--- a/daemon/unit_test.c
+++ b/daemon/unit_test.c
@@ -97,7 +97,7 @@ static int check_rrdcalc_comparisons(void) {
int check_storage_number(calculated_number n, int debug) {
char buffer[100];
- uint32_t flags = SN_EXISTS;
+ uint32_t flags = SN_DEFAULT_FLAGS;
storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s);
@@ -150,7 +150,7 @@ calculated_number storage_number_min(calculated_number n) {
do {
last = n;
n /= 2.0;
- storage_number t = pack_storage_number(n, SN_EXISTS);
+ storage_number t = pack_storage_number(n, SN_DEFAULT_FLAGS);
r = unpack_storage_number(t);
} while(r != 0.0 && r != last);
@@ -263,7 +263,7 @@ void benchmark_storage_number(int loop, int multiplier) {
n *= multiplier;
if(n > storage_number_positive_max) n = storage_number_positive_min;
- s = pack_storage_number(n, SN_EXISTS);
+ s = pack_storage_number(n, SN_DEFAULT_FLAGS);
d = unpack_storage_number(s);
print_calculated_number(buffer, d);
}
@@ -289,25 +289,12 @@ void benchmark_storage_number(int loop, int multiplier) {
}
static int check_storage_number_exists() {
- uint32_t flags;
-
-
- for(flags = 0; flags < 7 ; flags++) {
- if(get_storage_number_flags(flags << 24) != flags << 24) {
- fprintf(stderr, "Flag 0x%08x is not checked correctly. It became 0x%08x\n", flags << 24, get_storage_number_flags(flags << 24));
- return 1;
- }
- }
-
- flags = SN_EXISTS;
+ uint32_t flags = SN_DEFAULT_FLAGS;
calculated_number n = 0.0;
storage_number s = pack_storage_number(n, flags);
calculated_number d = unpack_storage_number(s);
- if(get_storage_number_flags(s) != flags) {
- fprintf(stderr, "Wrong flags. Given %08x, Got %08x!\n", flags, get_storage_number_flags(s));
- return 1;
- }
+
if(n != d) {
fprintf(stderr, "Wrong number returned. Expected " CALCULATED_NUMBER_FORMAT ", returned " CALCULATED_NUMBER_FORMAT "!\n", n, d);
return 1;
@@ -1192,7 +1179,7 @@ int run_test(struct test *test)
unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries;
for(c = 0 ; c < max ; c++) {
calculated_number v = unpack_storage_number(rd->values[c]);
- calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_EXISTS));
+ calculated_number n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));
int same = (calculated_number_round(v * 10000000.0) == calculated_number_round(n * 10000000.0))?1:0;
fprintf(stderr, " %s/%s: checking position %lu (at %lu secs), expecting value " CALCULATED_NUMBER_FORMAT ", found " CALCULATED_NUMBER_FORMAT ", %s\n",
test->name, rd->name, c+1,
@@ -1667,7 +1654,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
for (k = 0; k < QUERY_BATCH; ++k) {
last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
j * REGION_POINTS[current_region] + c + k;
- expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS));
+ expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));
n = rd[i][j]->state->query_ops.next_metric(&handle, &time_retrieved);
value = unpack_storage_number(n);
@@ -1725,7 +1712,7 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS]
assert(rd[i][j] == d);
last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c;
- expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS));
+ expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));
same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;
if(!same) {
@@ -1847,7 +1834,7 @@ int test_dbengine(void)
assert(rd[i][j] == d);
collected_number last = i * DIMS * REGION_POINTS[current_region] + j * REGION_POINTS[current_region] + c - point_offset;
- calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_EXISTS));
+ calculated_number expected = unpack_storage_number(pack_storage_number((calculated_number)last, SN_DEFAULT_FLAGS));
uint8_t same = (calculated_number_round(value) == calculated_number_round(expected)) ? 1 : 0;
if(!same) {
@@ -2076,7 +2063,7 @@ static void query_dbengine_chart(void *arg)
++thread_info->queries_nr;
for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
generatedv = generate_dbengine_chart_value(i, j, time_now);
- expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_EXISTS));
+ expected = unpack_storage_number(pack_storage_number((calculated_number) generatedv, SN_DEFAULT_FLAGS));
if (unlikely(rd->state->query_ops.is_finished(&handle))) {
if (!thread_info->delete_old_data) { /* data validation only when we don't delete */
diff --git a/database/engine/rrdengine.c b/database/engine/rrdengine.c
index f48181d29a..54a9cdf8dc 100644
--- a/database/engine/rrdengine.c
+++ b/database/engine/rrdengine.c
@@ -336,7 +336,7 @@ after_crc_check:
/* care, we don't hold the descriptor mutex */
if (have_read_error) {
/* Applications should make sure NULL values match 0 as does SN_EMPTY_SLOT */
- memset(page, 0, descr->page_length);
+ memset(page, SN_EMPTY_SLOT, descr->page_length);
} else if (RRD_NO_COMPRESSION == header->compression_algorithm) {
(void) memcpy(page, xt_io_descr->buf + payload_offset + page_offset, descr->page_length);
} else {
diff --git a/database/rrddim.c b/database/rrddim.c
index 51c4428c72..5479ade7aa 100644
--- a/database/rrddim.c
+++ b/database/rrddim.c
@@ -119,7 +119,7 @@ inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor)
// RRDDIM legacy data collection functions
static void rrddim_collect_init(RRDDIM *rd) {
- rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS);
+ rd->values[rd->rrdset->current_entry] = SN_EMPTY_SLOT;
}
static void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storage_number number) {
(void)point_in_time;
diff --git a/database/rrdset.c b/database/rrdset.c
index e4701e3b9e..282b0e5ba1 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -1120,7 +1120,7 @@ static inline size_t rrdset_done_interpolate(
, usec_t last_collect_ut
, usec_t now_collect_ut
, char store_this_entry
- , uint32_t storage_flags
+ , uint32_t has_reset_value
) {
RRDDIM *rd;
@@ -1135,6 +1135,11 @@ static inline size_t rrdset_done_interpolate(
size_t counter = st->counter;
long current_entry = st->current_entry;
+ uint32_t storage_flags = SN_DEFAULT_FLAGS;
+
+ if (has_reset_value)
+ storage_flags |= SN_EXISTS_RESET;
+
for( ; next_store_ut <= now_collect_ut ; last_collect_ut = next_store_ut, next_store_ut += update_every_ut, iterations-- ) {
#ifdef NETDATA_INTERNAL_CHECKS
@@ -1232,8 +1237,8 @@ static inline size_t rrdset_done_interpolate(
}
if(unlikely(!store_this_entry)) {
- rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); //pack_storage_number(0, SN_NOT_EXISTS)
-// rd->values[current_entry] = SN_EMPTY_SLOT; //pack_storage_number(0, SN_NOT_EXISTS);
+ rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT);
+// rd->values[current_entry] = SN_EMPTY_SLOT;
continue;
}
@@ -1261,8 +1266,8 @@ static inline size_t rrdset_done_interpolate(
);
#endif
-// rd->values[current_entry] = SN_EMPTY_SLOT; // pack_storage_number(0, SN_NOT_EXISTS);
- rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT); //pack_storage_number(0, SN_NOT_EXISTS)
+// rd->values[current_entry] = SN_EMPTY_SLOT;
+ rd->state->collect_ops.store_metric(rd, next_store_ut, SN_EMPTY_SLOT);
rd->last_stored_value = NAN;
}
@@ -1274,11 +1279,10 @@ static inline size_t rrdset_done_interpolate(
calculated_number t2 = unpack_storage_number(rd->values[current_entry]);
calculated_number accuracy = accuracy_loss(t1, t2);
- debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
+ debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
, st->id, rd->name
, current_entry
, t2
- , get_storage_number_flags(rd->values[current_entry])
, t1
, accuracy
, (accuracy > ACCURACY_LOSS_ACCEPTED_PERCENT) ? " **TOO BIG** " : ""
@@ -1300,7 +1304,7 @@ static inline size_t rrdset_done_interpolate(
#endif
}
// reset the storage flags for the next point, if any;
- storage_flags = SN_EXISTS;
+ storage_flags = SN_DEFAULT_FLAGS;
st->counter = ++counter;
st->current_entry = current_entry = ((current_entry + 1) >= st->entries) ? 0 : current_entry + 1;
@@ -1540,7 +1544,7 @@ after_first_database_work:
st->collected_total += rd->collected_value;
}
- uint32_t storage_flags = SN_EXISTS;
+ uint32_t has_reset_value = 0;
// process all dimensions to calculate their values
// based on the collected figures only
@@ -1637,7 +1641,7 @@ after_first_database_work:
, rd->collected_value);
if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)))
- storage_flags = SN_EXISTS_RESET;
+ has_reset_value = 1;
uint64_t last = (uint64_t)rd->last_collected_value;
uint64_t new = (uint64_t)rd->collected_value;
@@ -1708,7 +1712,7 @@ after_first_database_work:
);
if(!(rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)))
- storage_flags = SN_EXISTS_RESET;
+ has_reset_value = 1;
rd->last_collected_value = rd->collected_value;
}
@@ -1787,7 +1791,7 @@ after_first_database_work:
, last_collect_ut
, now_collect_ut
, store_this_entry
- , storage_flags
+ , has_reset_value
);
after_second_database_work:
diff --git a/exporting/tests/test_exporting_engine.c b/exporting/tests/test_exporting_engine.c
index 845c96c3c9..7188c6eeeb 100644
--- a/exporting/tests/test_exporting_engine.c
+++ b/exporting/tests/test_exporting_engine.c
@@ -312,12 +312,12 @@ static void test_exporting_calculate_value_from_stored_data(void **state)
expect_function_call(__mock_rrddim_query_is_finished);
will_return(__mock_rrddim_query_is_finished, 0);
expect_function_call(__mock_rrddim_query_next_metric);
- will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_EXISTS));
+ will_return(__mock_rrddim_query_next_metric, pack_storage_number(27, SN_DEFAULT_FLAGS));
expect_function_call(__mock_rrddim_query_is_finished);
will_return(__mock_rrddim_query_is_finished, 0);
expect_function_call(__mock_rrddim_query_next_metric);
- will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_EXISTS));
+ will_return(__mock_rrddim_query_next_metric, pack_storage_number(45, SN_DEFAULT_FLAGS));
expect_function_call(__mock_rrddim_query_is_finished);
will_return(__mock_rrddim_query_is_finished, 1);
@@ -431,7 +431,7 @@ static void test_format_dimension_stored_graphite_plaintext(void **state)
struct engine *engine = *state;
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
RRDDIM *rd = localhost->rrdset_root->dimensions;
assert_int_equal(format_dimension_stored_graphite_plaintext(engine->instance_root, rd), 0);
@@ -459,7 +459,7 @@ static void test_format_dimension_stored_json_plaintext(void **state)
struct engine *engine = *state;
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
RRDDIM *rd = localhost->rrdset_root->dimensions;
assert_int_equal(format_dimension_stored_json_plaintext(engine->instance_root, rd), 0);
@@ -487,7 +487,7 @@ static void test_format_dimension_stored_opentsdb_telnet(void **state)
struct engine *engine = *state;
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
RRDDIM *rd = localhost->rrdset_root->dimensions;
assert_int_equal(format_dimension_stored_opentsdb_telnet(engine->instance_root, rd), 0);
@@ -515,7 +515,7 @@ static void test_format_dimension_stored_opentsdb_http(void **state)
struct engine *engine = *state;
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
RRDDIM *rd = localhost->rrdset_root->dimensions;
assert_int_equal(format_dimension_stored_opentsdb_http(engine->instance_root, rd), 0);
@@ -1070,7 +1070,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
will_return(__wrap_now_realtime_sec, 2);
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(localhost, buffer, "test_server", "test_prefix", 0, 0);
@@ -1087,7 +1087,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
will_return(__wrap_now_realtime_sec, 2);
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
localhost, buffer, "test_server", "test_prefix", 0, PROMETHEUS_OUTPUT_NAMES | PROMETHEUS_OUTPUT_TYPES);
@@ -1106,7 +1106,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
will_return(__wrap_now_realtime_sec, 2);
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(localhost, buffer, "test_server", "test_prefix", 0, 0);
@@ -1265,7 +1265,7 @@ static void test_format_dimension_prometheus_remote_write(void **state)
RRDDIM *rd = localhost->rrdset_root->dimensions;
expect_function_call(__wrap_exporting_calculate_value_from_stored_data);
- will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_EXISTS));
+ will_return(__wrap_exporting_calculate_value_from_stored_data, pack_storage_number(27, SN_DEFAULT_FLAGS));
expect_function_call(__wrap_add_metric);
expect_value(__wrap_add_metric, write_request_p, 0xff);
diff --git a/libnetdata/storage_number/storage_number.c b/libnetdata/storage_number/storage_number.c
index 8ef1353b09..3e6a9f45c4 100644
--- a/libnetdata/storage_number/storage_number.c
+++ b/libnetdata/storage_number/storage_number.c
@@ -2,17 +2,23 @@
#include "../libnetdata.h"
+#define get_storage_number_flags(value) \
+ ((((storage_number)(value)) & (1 << 24)) | \
+ (((storage_number)(value)) & (1 << 25)) | \
+ (((storage_number)(value)) & (1 << 26)))
+
storage_number pack_storage_number(calculated_number value, uint32_t flags) {
// bit 32 = sign 0:positive, 1:negative
// bit 31 = 0:divide, 1:multiply
// bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
// bit 27 SN_EXISTS_100
// bit 26 SN_EXISTS_RESET
- // bit 25 SN_EXISTS
+ // bit 25 SN_ANOMALY_BIT = 0: anomalous, 1: not anomalous
// bit 24 to bit 1 = the value
storage_number r = get_storage_number_flags(flags);
- if(!value) return r;
+ if(!value)
+ goto RET_SN;
int m = 0;
calculated_number n = value, factor = 10;
@@ -47,7 +53,7 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) {
error("Number " CALCULATED_NUMBER_FORMAT " is too big.", value);
#endif
r += 0x00ffffff;
- return r;
+ goto RET_SN;
}
}
else {
@@ -78,6 +84,10 @@ storage_number pack_storage_number(calculated_number value, uint32_t flags) {
r += (storage_number)n;
#endif
+RET_SN:
+ if (r == SN_EMPTY_SLOT)
+ r = SN_ANOMALOUS_ZERO;
+
return r;
}
@@ -100,7 +110,7 @@ calculated_number unpack_storage_number(storage_number value) {
factor = 100;
// bit 26 SN_EXISTS_RESET
- // bit 25 SN_EXISTS
+ // bit 25 SN_ANOMALY_BIT
// bit 30, 29, 28 = (multiplier or divider) 0-7 (8 total)
int mul = (value & ((1<<29)|(1<<28)|(1<<27))) >> 27;
diff --git a/libnetdata/storage_number/storage_number.h b/libnetdata/storage_number/storage_number.h
index 4ad7ff6246..4101f69e01 100644
--- a/libnetdata/storage_number/storage_number.h
+++ b/libnetdata/storage_number/storage_number.h
@@ -60,17 +60,24 @@ typedef long double collected_number;
typedef uint32_t storage_number;
#define STORAGE_NUMBER_FORMAT "%u"
-#define SN_EXISTS (1 << 24) // the value exists
+#define SN_ANOMALY_BIT (1 << 24) // the anomaly bit of the value
#define SN_EXISTS_RESET (1 << 25) // the value has been overflown
#define SN_EXISTS_100 (1 << 26) // very large value (multiplier is 100 instead of 10)
-// extract the flags
-#define get_storage_number_flags(value) ((((storage_number)(value)) & (1 << 24)) | (((storage_number)(value)) & (1 << 25)) | (((storage_number)(value)) & (1 << 26)))
+#define SN_DEFAULT_FLAGS SN_ANOMALY_BIT
+
#define SN_EMPTY_SLOT 0x00000000
+// When the calculated number is zero and the value is anomalous (ie. it's bit
+// is zero) we want to return a storage_number representation that is
+// different from the empty slot. We achieve this by mapping zero to
+// SN_EXISTS_100. Unpacking the SN_EXISTS_100 value will return zero because
+// its fraction field (as well as its exponent factor field) will be zero.
+#define SN_ANOMALOUS_ZERO SN_EXISTS_100
+
// checks
-#define does_storage_number_exist(value) ((get_storage_number_flags(value) != 0)?1:0)
-#define did_storage_number_reset(value) ((get_storage_number_flags(value) == SN_EXISTS_RESET)?1:0)
+#define does_storage_number_exist(value) (((storage_number) (value)) != SN_EMPTY_SLOT)
+#define did_storage_number_reset(value) ((((storage_number) (value)) & SN_EXISTS_RESET) != 0)
storage_number pack_storage_number(calculated_number value, uint32_t flags);
calculated_number unpack_storage_number(storage_number value);
diff --git a/libnetdata/storage_number/tests/test_storage_number.c b/libnetdata/storage_number/tests/test_storage_number.c
index 7ef18b1de9..f90521cabe 100644
--- a/libnetdata/storage_number/tests/test_storage_number.c
+++ b/libnetdata/storage_number/tests/test_storage_number.c
@@ -38,7 +38,7 @@ static void test_number_printing(void **state)
print_calculated_number(value, -9999.9999999);
assert_string_equal(value, "-9999.9999999");
- print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_EXISTS)));
+ print_calculated_number(value, unpack_storage_number(pack_storage_number(16.777218L, SN_DEFAULT_FLAGS)));
assert_string_equal(value, "16.77722");
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-07 17:10:02 +0000