diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2022-05-03 00:31:19 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-03 00:31:19 +0300 |
commit | 87c0cc2d6049c46f38b9c866668a0a24a3e962c0 (patch) | |
tree | 066c18bc90894e89ce46dc0b2e104c61018e830c | |
parent | 47fa3d708902fb001b2e88e4145d2a451549cd8e (diff) |
One way allocator to double the speed of parallel context queries (#12787)
* one way allocator to speed up context queries
* fixed a bug while expanding memory pages
* reworked for clarity and finally fixed the bug of allocating memory beyond the page size
* further optimize allocation step to minimize the number of allocations made
* implement strdup with memcpy instead of strcpy
* added documentation
* prevent an uninitialized use of owa
* added callocz() interface
* integrate onewayalloc everywhere - apart sql queries
* one way allocator is now used in context queries using archived charts in sql
* align on the size of pointers
* forgotten freez()
* removed not needed memcpys
* give unique names to global variables to avoid conflicts with system definitions
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rw-r--r-- | Makefile.am | 2 | ||||
-rw-r--r-- | configure.ac | 1 | ||||
-rw-r--r-- | daemon/unit_test.c | 12 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.c | 38 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.h | 2 | ||||
-rw-r--r-- | libnetdata/Makefile.am | 1 | ||||
-rw-r--r-- | libnetdata/libnetdata.h | 1 | ||||
-rw-r--r-- | libnetdata/onewayalloc/Makefile.am | 8 | ||||
-rw-r--r-- | libnetdata/onewayalloc/README.md | 71 | ||||
-rw-r--r-- | libnetdata/onewayalloc/onewayalloc.c | 173 | ||||
-rw-r--r-- | libnetdata/onewayalloc/onewayalloc.h | 17 | ||||
-rw-r--r-- | web/api/formatters/rrd2json.c | 75 | ||||
-rw-r--r-- | web/api/formatters/rrd2json.h | 9 | ||||
-rw-r--r-- | web/api/queries/query.c | 25 | ||||
-rw-r--r-- | web/api/queries/rrdr.c | 24 | ||||
-rw-r--r-- | web/api/queries/rrdr.h | 5 | ||||
-rw-r--r-- | web/api/web_api_v1.c | 18 |
18 files changed, 389 insertions, 95 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index f3df9feff1..87d293ddb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -383,6 +383,8 @@ set(LIBNETDATA_FILES libnetdata/log/log.h libnetdata/os.c libnetdata/os.h + libnetdata/onewayalloc/onewayalloc.c + libnetdata/onewayalloc/onewayalloc.h libnetdata/popen/popen.c libnetdata/popen/popen.h libnetdata/procfile/procfile.c diff --git a/Makefile.am b/Makefile.am index 9f60349588..7eacb49d0e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -158,6 +158,8 @@ LIBNETDATA_FILES = \ libnetdata/locks/locks.h \ libnetdata/log/log.c \ libnetdata/log/log.h \ + libnetdata/onewayalloc/onewayalloc.c \ + libnetdata/onewayalloc/onewayalloc.h \ libnetdata/popen/popen.c \ libnetdata/popen/popen.h \ libnetdata/procfile/procfile.c \ diff --git a/configure.ac b/configure.ac index 914a871a8b..9e3a00f855 100644 --- a/configure.ac +++ b/configure.ac @@ -1758,6 +1758,7 @@ AC_CONFIG_FILES([ libnetdata/eval/Makefile libnetdata/locks/Makefile libnetdata/log/Makefile + libnetdata/onewayalloc/Makefile libnetdata/popen/Makefile libnetdata/procfile/Makefile libnetdata/simple_pattern/Makefile diff --git a/daemon/unit_test.c b/daemon/unit_test.c index 7a52735d50..2fc16e2d07 100644 --- a/daemon/unit_test.c +++ b/daemon/unit_test.c @@ -1732,7 +1732,8 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] update_every = REGION_UPDATE_EVERY[current_region]; long points = (time_end - time_start) / update_every; for (i = 0 ; i < CHARTS ; ++i) { - RRDR *r = rrd2rrdr(st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0); + ONEWAYALLOC *owa = onewayalloc_create(0); + RRDR *r = rrd2rrdr(owa, st[i], points, time_start + update_every, time_end, RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0); if (!r) { fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name); return ++errors; @@ -1766,8 +1767,9 @@ static int test_dbengine_check_rrdr(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DIMS] } } } - rrdr_free(r); + rrdr_free(owa, r); } + onewayalloc_destroy(owa); } return errors; } @@ -1851,7 +1853,8 @@ int test_dbengine(void) long points = (time_end[REGIONS - 1] - time_start[0]) / update_every; // cover all time regions with RRDR long point_offset = (time_start[current_region] - time_start[0]) / update_every; for (i = 0 ; i < CHARTS ; ++i) { - RRDR *r = rrd2rrdr(st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0); + ONEWAYALLOC *owa = onewayalloc_create(0); + RRDR *r = rrd2rrdr(owa, st[i], points, time_start[0] + update_every, time_end[REGIONS - 1], RRDR_GROUPING_AVERAGE, 0, 0, NULL, NULL, 0); if (!r) { fprintf(stderr, " DB-engine unittest %s: empty RRDR ### E R R O R ###\n", st[i]->name); ++errors; @@ -1888,8 +1891,9 @@ int test_dbengine(void) } } } - rrdr_free(r); + rrdr_free(owa, r); } + onewayalloc_destroy(owa); } error_out: rrd_wrlock(); diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index b22350d180..a69c12af8d 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -1446,13 +1446,13 @@ int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id } #ifdef ENABLE_DBENGINE -static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid) +static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *name, uuid_t *metric_uuid) { - RRDDIM *rd = callocz(1, sizeof(*rd)); + RRDDIM *rd = onewayalloc_callocz(owa, 1, sizeof(*rd)); rd->rrdset = st; rd->last_stored_value = NAN; rrddim_flag_set(rd, RRDDIM_FLAG_NONE); - rd->state = mallocz(sizeof(*rd->state)); + rd->state = onewayalloc_mallocz(owa, sizeof(*rd->state)); rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; rd->state->query_ops.init = rrdeng_load_metric_init; rd->state->query_ops.next_metric = rrdeng_load_metric_next; @@ -1460,11 +1460,11 @@ static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metr rd->state->query_ops.finalize = rrdeng_load_metric_finalize; rd->state->query_ops.latest_time = rrdeng_metric_latest_time; rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time; - rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t)); + rd->state->rrdeng_uuid = onewayalloc_mallocz(owa, sizeof(uuid_t)); uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid); uuid_copy(rd->state->metric_uuid, *metric_uuid); - rd->id = strdupz(id); - rd->name = strdupz(name); + rd->id = onewayalloc_strdupz(owa, id); + rd->name = onewayalloc_strdupz(owa, name); return rd; } #endif @@ -1481,7 +1481,7 @@ static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metr "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \ "order by c.chart_id asc, c.type||'.'||c.id desc;" -void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart) +void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart) { #ifdef ENABLE_DBENGINE int rc; @@ -1490,7 +1490,7 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho return; if (unlikely(!(*param_list))) { - *param_list = mallocz(sizeof(struct context_param)); + *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param)); (*param_list)->first_entry_t = LONG_MAX; (*param_list)->last_entry_t = 0; (*param_list)->rd = NULL; @@ -1539,21 +1539,21 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) { if (unlikely(st && !st->counter)) { - freez(st->context); - freez((char *) st->name); - freez(st); + onewayalloc_freez(owa, st->context); + onewayalloc_freez(owa, (char *) st->name); + onewayalloc_freez(owa, st); } - st = callocz(1, sizeof(*st)); + st = onewayalloc_callocz(owa, 1, sizeof(*st)); char n[RRD_ID_LENGTH_MAX + 1]; snprintfz( n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4), (char *)sqlite3_column_text(res, 3)); - st->name = strdupz(n); + st->name = onewayalloc_strdupz(owa, n); st->update_every = sqlite3_column_int(res, 6); st->counter = 0; if (chart) { - st->context = strdupz((char *)sqlite3_column_text(res, 8)); + st->context = onewayalloc_strdupz(owa, (char *)sqlite3_column_text(res, 8)); strncpyz(st->id, chart, RRD_ID_LENGTH_MAX); } uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7)); @@ -1569,7 +1569,7 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho st->counter++; st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t); - RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid); + RRDDIM *rd = create_rrdim_entry(owa, st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid); if (sqlite3_column_int(res, 9) == 1) rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN); rd->next = (*param_list)->rd; @@ -1577,13 +1577,13 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho } if (st) { if (!st->counter) { - freez(st->context); - freez((char *)st->name); - freez(st); + onewayalloc_freez(owa,st->context); + onewayalloc_freez(owa,(char *)st->name); + onewayalloc_freez(owa,st); } else if (!st->context && context) - st->context = strdupz(context); + st->context = onewayalloc_strdupz(owa,context); } failed: diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index 20b8b75757..286df9180a 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -89,7 +89,7 @@ extern void db_unlock(void); extern void db_lock(void); extern void delete_dimension_uuid(uuid_t *dimension_uuid); extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value); -extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart); +extern void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart); extern void store_claim_id(uuid_t *host_id, uuid_t *claim_id); extern int update_node_id(uuid_t *host_id, uuid_t *node_id); extern int get_node_id(uuid_t *host_id, uuid_t *node_id); diff --git a/libnetdata/Makefile.am b/libnetdata/Makefile.am index e787801c28..ea24601498 100644 --- a/libnetdata/Makefile.am +++ b/libnetdata/Makefile.am @@ -17,6 +17,7 @@ SUBDIRS = \ health \ locks \ log \ + onewayalloc \ popen \ procfile \ simple_pattern \ diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index f937a1bf9c..d197f3f7c6 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -345,6 +345,7 @@ extern char *netdata_configured_host_prefix; #include "json/json.h" #include "health/health.h" #include "string/utf8.h" +#include "onewayalloc/onewayalloc.h" // BEWARE: Outside of the C code this also exists in alarm-notify.sh #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud" diff --git a/libnetdata/onewayalloc/Makefile.am b/libnetdata/onewayalloc/Makefile.am new file mode 100644 index 0000000000..161784b8f6 --- /dev/null +++ b/libnetdata/onewayalloc/Makefile.am @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in + +dist_noinst_DATA = \ + README.md \ + $(NULL) diff --git a/libnetdata/onewayalloc/README.md b/libnetdata/onewayalloc/README.md new file mode 100644 index 0000000000..1f459c2636 --- /dev/null +++ b/libnetdata/onewayalloc/README.md @@ -0,0 +1,71 @@ +<!-- +title: "One Way Allocator" +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/onewayallocator/README.md +--> + +# One Way Allocator + +This is a very fast single-threaded-only memory allocator, that minimized system calls +when a lot of memory allocations needs to be made to perform a task, which all of them +can be freed together when the task finishes. + +It has been designed to be used for netdata context queries. + +For netdata to perform a context query, it builds a virtual chart, a chart that contains +all the dimensions of the charts having the same context. This process requires allocating +several structures for each of the dimensions to attach them to the virtual chart. All +these data can be freed immediately after the query finishes. + +## How it works + +1. The caller calls `ONEWAYALLOC *owa = onewayalloc_create(sizehint)` to create an OWA. + Internally this allocates the first memory buffer with size >= `sizehint`. + If `sizehint` is zero, it will allocate 1 hardware page (usually 4kb). + No need to check for success or failure. As with `mallocz()` in netdata, a `fatal()` + will be called if the allocation fails - although this will never fail, since Linux + does not really check if there is memory available for `mmap()` calls. + +2. The caller can then perform any number of the following calls to acquire memory: + - `onewayalloc_mallocz(owa, size)`, similar to `mallocz()` + - `onewayalloc_callocz(owa, nmemb, size)`, similar to `callocz()` + - `onewayalloc_strdupz(owa, string)`, similar to `strdupz()` + - `onewayalloc_memdupz(owa, ptr, size)`, similar to `mallocz()` and then `memcpy()` + +3. Once the caller has done all the work with the allocated buffers, all memory allocated + can be freed with `onewayalloc_destroy(owa)`. + +## How faster it is? + +On modern hardware, for any single query the performance improvement is marginal and not +noticeable at all. + +We performed the following tests using the same huge context query (1000 charts, +100 dimensions each = 100k dimensions) + +1. using `mallocz()`, 1 caller, 256 queries (sequential) +2. using `mallocz()`, 256 callers, 1 query each (parallel) +3. using `OWA`, 1 caller, 256 queries (sequential) +4. using `OWA`, 256 callers, 1 query each (parallel) + +Netdata was configured to use 24 web threads on the 24 core server we used. + +The results are as follows: + +### sequential test + +branch|transactions|time to complete|transaction rate|average response time|min response time|max response time +:---:|:---:|:---:|:---:|:---:|:---:|:---:| +`malloc()`|256|322.35s|0.79/sec|1.26s|1.01s|1.87s +`OWA`|256|310.19s|0.83/sec|1.21s|1.04s|1.63s + +For a single query, the improvement is just marginal and not noticeable at all. + +### parallel test + +branch|transactions|time to complete|transaction rate|average response time|min response time|max response time +:---:|:---:|:---:|:---:|:---:|:---:|:---:| +`malloc()`|256|84.72s|3.02/sec|68.43s|50.20s|84.71s +`OWA`|256|39.35s|6.51/sec|34.48s|20.55s|39.34s + +For parallel workload, like the one executed by netdata.cloud, `OWA` provides a 54% overall speed improvement (more than double the overall +user-experienced speed, including the data query itself). diff --git a/libnetdata/onewayalloc/onewayalloc.c b/libnetdata/onewayalloc/onewayalloc.c new file mode 100644 index 0000000000..3bc6e4bc7c --- /dev/null +++ b/libnetdata/onewayalloc/onewayalloc.c @@ -0,0 +1,173 @@ +#include "onewayalloc.h" + +static size_t OWA_NATURAL_PAGE_SIZE = 0; +static size_t OWA_NATURAL_ALIGNMENT = sizeof(int*); + +typedef struct owa_page { + size_t stats_pages; + size_t stats_pages_size; + size_t stats_mallocs_made; + size_t stats_mallocs_size; + size_t size; // the total size of the page + size_t offset; // the first free byte of the page + struct owa_page *next; // the next page on the list + struct owa_page *last; // the last page on the list - we currently allocate on this +} OWA_PAGE; + +// allocations need to be aligned to CPU register width +// https://en.wikipedia.org/wiki/Data_structure_alignment +static inline size_t natural_alignment(size_t size) { + if(unlikely(size % OWA_NATURAL_ALIGNMENT)) + size = size + OWA_NATURAL_ALIGNMENT - (size % OWA_NATURAL_ALIGNMENT); + + return size; +} + +// Create an OWA +// Once it is created, the called may call the onewayalloc_mallocz() +// any number of times, for any amount of memory. + +static OWA_PAGE *onewayalloc_create_internal(OWA_PAGE *head, size_t size_hint) { + if(unlikely(!OWA_NATURAL_PAGE_SIZE)) + OWA_NATURAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE); + + // our default page size + size_t size = OWA_NATURAL_PAGE_SIZE; + + // make sure the new page will fit both the requested size + // and the OWA_PAGE structure at its beginning + size_hint += sizeof(OWA_PAGE); + + // prefer the user size if it is bigger than our size + if(size_hint > size) size = size_hint; + + // try to allocate half of the total we have allocated already + if(likely(head)) { + size_t optimal_size = head->stats_pages_size / 2; + if(optimal_size > size) size = optimal_size; + } + + // Make sure our allocations are always a multiple of the hardware page size + if(size % OWA_NATURAL_PAGE_SIZE) size = size + OWA_NATURAL_PAGE_SIZE - (size % OWA_NATURAL_PAGE_SIZE); + + OWA_PAGE *page = (OWA_PAGE *)netdata_mmap(NULL, size, MAP_ANONYMOUS|MAP_PRIVATE, 0); + if(unlikely(!page)) fatal("Cannot allocate onewayalloc buffer of size %zu", size); + + page->size = size; + page->offset = natural_alignment(sizeof(OWA_PAGE)); + page->next = page->last = NULL; + + if(unlikely(!head)) { + // this is the first time we are called + head = page; + head->stats_pages = 0; + head->stats_pages_size = 0; + head->stats_mallocs_made = 0; + head->stats_mallocs_size = 0; + } + else { + // link this page into our existing linked list + head->last->next = page; + } + + head->last = page; + head->stats_pages++; + head->stats_pages_size += size; + + return (ONEWAYALLOC *)page; +} + +ONEWAYALLOC *onewayalloc_create(size_t size_hint) { + return onewayalloc_create_internal(NULL, size_hint); +} + +void *onewayalloc_mallocz(ONEWAYALLOC *owa, size_t size) { + OWA_PAGE *head = (OWA_PAGE *)owa; + OWA_PAGE *page = head->last; + + // update stats + head->stats_mallocs_made++; + head->stats_mallocs_size += size; + + // make sure the size is aligned + size = natural_alignment(size); + + if(unlikely(page->size - page->offset < size)) { + // we don't have enough space to fit the data + // let's get another page + page = onewayalloc_create_internal(head, (size > page->size)?size:page->size); + } + + char *mem = (char *)page; + mem = &mem[page->offset]; + page->offset += size; + + return (void *)mem; +} + +void *onewayalloc_callocz(ONEWAYALLOC *owa, size_t nmemb, size_t size) { + size_t total = nmemb * size; + void *mem = onewayalloc_mallocz(owa, total); + memset(mem, 0, total); + return mem; +} + +char *onewayalloc_strdupz(ONEWAYALLOC *owa, const char *s) { + size_t size = strlen(s) + 1; + char *d = onewayalloc_mallocz((OWA_PAGE *)owa, size); + memcpy(d, s, size); + return d; +} + +void *onewayalloc_memdupz(ONEWAYALLOC *owa, const void *src, size_t size) { + void *mem = onewayalloc_mallocz((OWA_PAGE *)owa, size); + // memcpy() is way faster than strcpy() since it does not check for '\0' + memcpy(mem, src, size); + return mem; +} + +void onewayalloc_freez(ONEWAYALLOC *owa __maybe_unused, const void *ptr __maybe_unused) { +#ifdef NETDATA_INTERNAL_CHECKS + // allow the caller to call us for a mallocz() allocation + // so try to find it in our memory and if it is not there + // log an error + + OWA_PAGE *head = (OWA_PAGE *)owa; + OWA_PAGE *page; + size_t seeking = (size_t)ptr; + + for(page = head; page ;page = page->next) { + size_t start = (size_t)page; + size_t end = start + page->size; + + if(seeking >= start && seeking <= end) { + // found it - it is ours + // just return to let the caller think we actually did something + return; + } + } + + // not found - it is not ours + // let's free it with the system allocator + error("ONEWAYALLOC: request to free address 0x%p that is not allocated by this OWA", ptr); +#endif + + return; +} + +void onewayalloc_destroy(ONEWAYALLOC *owa) { + if(!owa) return; + + OWA_PAGE *head = (OWA_PAGE *)owa; + + //info("OWA: %zu allocations of %zu total bytes, in %zu pages of %zu total bytes", + // head->stats_mallocs_made, head->stats_mallocs_size, + // head->stats_pages, head->stats_pages_size); + + OWA_PAGE *page = head; + while(page) { + OWA_PAGE *p = page; + page = page->next; + munmap(p, p->size); + } +} diff --git a/libnetdata/onewayalloc/onewayalloc.h b/libnetdata/onewayalloc/onewayalloc.h new file mode 100644 index 0000000000..ed8f12f39b --- /dev/null +++ b/libnetdata/onewayalloc/onewayalloc.h @@ -0,0 +1,17 @@ +#ifndef ONEWAYALLOC_H +#define ONEWAYALLOC_H 1 + +#include "../libnetdata.h" + +typedef void ONEWAYALLOC; + +extern ONEWAYALLOC *onewayalloc_create(size_t size_hint); +extern void onewayalloc_destroy(ONEWAYALLOC *owa); + +extern void *onewayalloc_mallocz(ONEWAYALLOC *owa, size_t size); +extern void *onewayalloc_callocz(ONEWAYALLOC *owa, size_t nmemb, size_t size); +extern char *onewayalloc_strdupz(ONEWAYALLOC *owa, const char *s); +extern void *onewayalloc_memdupz(ONEWAYALLOC *owa, const void *src, size_t size); +extern void onewayalloc_freez(ONEWAYALLOC *owa, const void *ptr); + +#endif // ONEWAYALLOC_H diff --git a/web/api/formatters/rrd2json.c b/web/api/formatters/rrd2json.c index 1a8b07c7c8..9ac54f7589 100644 --- a/web/api/formatters/rrd2json.c +++ b/web/api/formatters/rrd2json.c @@ -2,27 +2,27 @@ #include "web/api/web_api_v1.h" -static inline void free_single_rrdrim(RRDDIM *temp_rd, int archive_mode) +static inline void free_single_rrdrim(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) { if (unlikely(!temp_rd)) return; - freez((char *)temp_rd->id); - freez((char *)temp_rd->name); + onewayalloc_freez(owa, (char *)temp_rd->id); if (unlikely(archive_mode)) { temp_rd->rrdset->counter--; if (!temp_rd->rrdset->counter) { - freez((char *)temp_rd->rrdset->name); - freez(temp_rd->rrdset->context); - freez(temp_rd->rrdset); + onewayalloc_freez(owa, (char *)temp_rd->rrdset->name); + onewayalloc_freez(owa, temp_rd->rrdset->context); + onewayalloc_freez(owa, temp_rd->rrdset); } } - freez(temp_rd->state); - freez(temp_rd); + + onewayalloc_freez(owa, temp_rd->state); + onewayalloc_freez(owa, temp_rd); } -static inline void free_rrddim_list(RRDDIM *temp_rd, int archive_mode) +static inline void free_rrddim_list(ONEWAYALLOC *owa, RRDDIM *temp_rd, int archive_mode) { if (unlikely(!temp_rd)) return; @@ -30,22 +30,22 @@ static inline void free_rrddim_list(RRDDIM *temp_rd, int archive_mode) RRDDIM *t; while (temp_rd) { t = temp_rd->next; - free_single_rrdrim(temp_rd, archive_mode); + free_single_rrdrim(owa, temp_rd, archive_mode); temp_rd = t; } } -void free_context_param_list(struct context_param **param_list) +void free_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list) { if (unlikely(!param_list || !*param_list)) return; - free_rrddim_list(((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE); - freez((*param_list)); + free_rrddim_list(owa, ((*param_list)->rd), (*param_list)->flags & CONTEXT_FLAGS_ARCHIVE); + onewayalloc_freez(owa, (*param_list)); *param_list = NULL; } -void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested) +void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested) { RRDDIM *temp_rd = context_param_list->rd; RRDDIM *new_rd_list = NULL, *t; @@ -59,19 +59,19 @@ void rebuild_context_param_list(struct context_param *context_param_list, time_t temp_rd->next = new_rd_list; new_rd_list = temp_rd; } else - free_single_rrdrim(temp_rd, is_archived); + free_single_rrdrim(owa, temp_rd, is_archived); temp_rd = t; } context_param_list->rd = new_rd_list; }; -void build_context_param_list(struct context_param **param_list, RRDSET *st) +void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st) { if (unlikely(!param_list || !st)) return; if (unlikely(!(*param_list))) { - *param_list = mallocz(sizeof(struct context_param)); + *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param)); (*param_list)->first_entry_t = LONG_MAX; (*param_list)->last_entry_t = 0; (*param_list)->flags = CONTEXT_FLAGS_CONTEXT; @@ -86,14 +86,10 @@ void build_context_param_list(struct context_param **param_list, RRDSET *st) (*param_list)->last_entry_t = MAX((*param_list)->last_entry_t, rrdset_last_entry_t_nolock(st)); rrddim_foreach_read(rd1, st) { - RRDDIM *rd = mallocz(rd1->memsize); - memcpy(rd, rd1, rd1->memsize); - rd->id = strdupz(rd1->id); - rd->name = strdupz(rd1->name); - rd->state = mallocz(sizeof(*rd->state)); - memcpy(rd->state, rd1->state, sizeof(*rd->state)); - memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops)); - memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops)); + RRDDIM *rd = onewayalloc_memdupz(owa, rd1, rd1->memsize); + rd->id = onewayalloc_strdupz(owa, rd1->id); + rd->name = onewayalloc_strdupz(owa, rd1->name); + rd->state = onewayalloc_memdupz(owa, rd1->state, sizeof(*rd->state)); rd->next = (*param_list)->rd; (*param_list)->rd = rd; } @@ -169,22 +165,27 @@ int rrdset2value_api_v1( , int *value_is_null , int timeout ) { + int ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + + ONEWAYALLOC *owa = onewayalloc_create(0); - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout); + RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions, NULL, timeout); if(!r) { if(value_is_null) *value_is_null = 1; - return HTTP_RESP_INTERNAL_SERVER_ERROR; + ret = HTTP_RESP_INTERNAL_SERVER_ERROR; + goto cleanup; } if(rrdr_rows(r) == 0) { - rrdr_free(r); + rrdr_free(owa, r); if(db_after) *db_after = 0; if(db_before) *db_before = 0; if(value_is_null) *value_is_null = 1; - return HTTP_RESP_BAD_REQUEST; + ret = HTTP_RESP_BAD_REQUEST; + goto cleanup; } if(wb) { @@ -199,13 +200,17 @@ int rrdset2value_api_v1( long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0; *n = rrdr2value(r, i, options, value_is_null, NULL); + ret = HTTP_RESP_OK; - rrdr_free(r); - return HTTP_RESP_OK; +cleanup: + if(r) rrdr_free(owa, r); + onewayalloc_destroy(owa); + return ret; } int rrdset2anything_api_v1( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , BUFFER *wb , BUFFER *dimensions , uint32_t format @@ -225,14 +230,14 @@ int rrdset2anything_api_v1( if (context_param_list && !(context_param_list->flags & CONTEXT_FLAGS_ARCHIVE)) st->last_accessed_time = now_realtime_sec(); - RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout); + RRDR *r = rrd2rrdr(owa, st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list, timeout); if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); return HTTP_RESP_INTERNAL_SERVER_ERROR; } if (r->result_options & RRDR_RESULT_OPTION_CANCEL) { - rrdr_free(r); + rrdr_free(owa, r); return HTTP_RESP_BACKEND_FETCH_FAILED; } @@ -411,6 +416,6 @@ int rrdset2anything_api_v1( break; } - rrdr_free(r); + rrdr_free(owa, r); return HTTP_RESP_OK; } diff --git a/web/api/formatters/rrd2json.h b/web/api/formatters/rrd2json.h index af809c54f9..a3fe50cbbb 100644 --- a/web/api/formatters/rrd2json.h +++ b/web/api/formatters/rrd2json.h @@ -54,7 +54,8 @@ extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb); extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format); extern int rrdset2anything_api_v1( - RRDSET *st + ONEWAYALLOC *owa + , RRDSET *st , BUFFER *wb , BUFFER *dimensions , uint32_t format @@ -88,8 +89,8 @@ extern int rrdset2value_api_v1( , int timeout ); -extern void build_context_param_list(struct context_param **param_list, RRDSET *st); -extern void rebuild_context_param_list(struct context_param *context_param_list, time_t after_requested); -extern void free_context_param_list(struct context_param **param_list); +extern void build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDSET *st); +extern void rebuild_context_param_list(ONEWAYALLOC *owa, struct context_param *context_param_list, time_t after_requested); +extern void free_context_param_li |