diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2022-05-03 00:30:21 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-03 00:30:21 +0300 |
commit | 47fa3d708902fb001b2e88e4145d2a451549cd8e (patch) | |
tree | d02bead8e9793a8742f95d7de2786c3a9d9d23a4 /libnetdata | |
parent | d6b1756ea7c0dc28b338aff67800e932d8fe97ec (diff) |
Speed up BUFFER increases (minimize reallocs) (#12792)
* speedup BUFFER increases by forward looking reallocs
* implemented buffer_vsprintf() and optimized buffer_sprintf() to minimize calls to vsnprintfz()
* optimize json generation for well known strings
Diffstat (limited to 'libnetdata')
-rw-r--r-- | libnetdata/buffer/buffer.c | 67 | ||||
-rw-r--r-- | libnetdata/buffer/buffer.h | 1 |
2 files changed, 49 insertions, 19 deletions
diff --git a/libnetdata/buffer/buffer.c b/libnetdata/buffer/buffer.c index 8ea90985c3..880417551b 100644 --- a/libnetdata/buffer/buffer.c +++ b/libnetdata/buffer/buffer.c @@ -136,6 +136,24 @@ void buffer_print_llu(BUFFER *wb, unsigned long long uvalue) wb->len += wstr - str; } +void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) { + if(unlikely(!txt || !*txt)) return; + + buffer_need_bytes(wb, len + 1); + + char *s = &wb->buffer[wb->len]; + const char *end = &txt[len + 1]; + + while(txt != end) + *s++ = *txt++; + + wb->len += len; + + // keep it NULL terminating + // not counting it at wb->len + wb->buffer[wb->len] = '\0'; +} + void buffer_strcat(BUFFER *wb, const char *txt) { // buffer_sprintf(wb, "%s", txt); @@ -159,8 +177,7 @@ void buffer_strcat(BUFFER *wb, const char *txt) if(*txt) { debug(D_WEB_BUFFER, "strcat(): increasing web_buffer at position %zu, size = %zu\n", wb->len, wb->size); len = strlen(txt); - buffer_increase(wb, len); - buffer_strcat(wb, txt); + buffer_fast_strcat(wb, txt, len); } else { // terminate the string @@ -236,15 +253,23 @@ void buffer_vsprintf(BUFFER *wb, const char *fmt, va_list args) { if(unlikely(!fmt || !*fmt)) return; - buffer_need_bytes(wb, 2); + size_t wrote = 0, need = 2, space_remaining = 0; - size_t len = wb->size - wb->len - 1; + do { + need += space_remaining * 2; - wb->len += vsnprintfz(&wb->buffer[wb->len], len, fmt, args); + debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); + buffer_need_bytes(wb, need); - buffer_overflow_check(wb); + space_remaining = wb->size - wb->len - 1; - // the buffer is \0 terminated by vsnprintfz + wrote = (size_t) vsnprintfz(&wb->buffer[wb->len], space_remaining, fmt, args); + + } while(wrote >= space_remaining); + + wb->len += wrote; + + // the buffer is \0 terminated by vsnprintf } void buffer_sprintf(BUFFER *wb, const char *fmt, ...) @@ -252,22 +277,21 @@ void buffer_sprintf(BUFFER *wb, const char *fmt, ...) if(unlikely(!fmt || !*fmt)) return; va_list args; - size_t wrote = 0, need = 2, multiplier = 0, len; + size_t wrote = 0, need = 2, space_remaining = 0; do { - need += wrote + multiplier * WEB_DATA_LENGTH_INCREASE_STEP; - multiplier++; + need += space_remaining * 2; debug(D_WEB_BUFFER, "web_buffer_sprintf(): increasing web_buffer at position %zu, size = %zu, by %zu bytes (wrote = %zu)\n", wb->len, wb->size, need, wrote); buffer_need_bytes(wb, need); - len = wb->size - wb->len - 1; + space_remaining = wb->size - wb->len - 1; va_start(args, fmt); - wrote = (size_t) vsnprintfz(&wb->buffer[wb->len], len, fmt, args); + wrote = (size_t) vsnprintfz(&wb->buffer[wb->len], space_remaining, fmt, args); va_end(args); - } while(wrote >= len); + } while(wrote >= space_remaining); wb->len += wrote; @@ -420,16 +444,21 @@ void buffer_increase(BUFFER *b, size_t free_size_required) { buffer_overflow_check(b); size_t left = b->size - b->len; - if(left >= free_size_required) return; - size_t increase = free_size_required - left; - if(increase < WEB_DATA_LENGTH_INCREASE_STEP) increase = WEB_DATA_LENGTH_INCREASE_STEP; + size_t wanted = free_size_required - left; + size_t minimum = WEB_DATA_LENGTH_INCREASE_STEP; + if(minimum > wanted) wanted = minimum; + + size_t optimal = b->size; + if(b->size > 5*1024*1024) optimal = b->size / 2; + + if(optimal > wanted) wanted = optimal; - debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + increase); + debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", b->size, b->size + wanted); - b->buffer = reallocz(b->buffer, b->size + increase + sizeof(BUFFER_OVERFLOW_EOF) + 2); - b->size += increase; + b->buffer = reallocz(b->buffer, b->size + wanted + sizeof(BUFFER_OVERFLOW_EOF) + 2); + b->size += wanted; buffer_overflow_init(b); buffer_overflow_check(b); diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h index d50910c51c..ceaeadd9b7 100644 --- a/libnetdata/buffer/buffer.h +++ b/libnetdata/buffer/buffer.h @@ -55,6 +55,7 @@ extern const char *buffer_tostring(BUFFER *wb); extern void buffer_reset(BUFFER *wb); extern void buffer_strcat(BUFFER *wb, const char *txt); +extern void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len); extern void buffer_rrd_value(BUFFER *wb, calculated_number value); extern void buffer_date(BUFFER *wb, int year, int month, int day, int hours, int minutes, int seconds); |