diff options
author | Costa Tsaousis <costa@netdata.cloud> | 2022-05-02 14:33:41 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-05-02 14:33:41 +0300 |
commit | 43b9fdc213ee043c38e248d9ff404ae02cae480d (patch) | |
tree | 5ce93c6a395bf78c1c131d2d90998898bab77ead /libnetdata | |
parent | f389f8c7f0f24b3cde2f56cfd8dc771020fbf7d3 (diff) |
procfile: more comfortable initial settings and faster/fewer reallocs (#12791)
Diffstat (limited to 'libnetdata')
-rw-r--r-- | libnetdata/libnetdata.c | 71 | ||||
-rw-r--r-- | libnetdata/libnetdata.h | 3 | ||||
-rw-r--r-- | libnetdata/procfile/procfile.c | 30 |
3 files changed, 53 insertions, 51 deletions
diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c index 36787042ef..4c38a3a5e7 100644 --- a/libnetdata/libnetdata.c +++ b/libnetdata/libnetdata.c @@ -31,6 +31,8 @@ const char *program_version = VERSION; // routines. #ifdef NETDATA_LOG_ALLOCATIONS +#warning NETDATA_LOG_ALLOCATIONS ENABLED - set log_thread_memory_allocations=1 on any thread to log all its allocations - or use log_allocations() to log them on demand + static __thread struct memory_statistics { volatile ssize_t malloc_calls_made; volatile ssize_t calloc_calls_made; @@ -44,23 +46,14 @@ static __thread struct memory_statistics { __thread size_t log_thread_memory_allocations = 0; -static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) { +inline void log_allocations_int(const char *file, const char *function, const unsigned long line) { static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 }; - fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n", - netdata_thread_tag(), - log_thread_memory_allocations, - line, file, function, - type, size - ); - - fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n", + fprintf(stderr, "%s MEMORY ALLOCATIONS: (%04lu@%s:%s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n", netdata_thread_tag(), - log_thread_memory_allocations, line, file, function, (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory, (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory, - type, memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made, memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made, memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made, @@ -79,12 +72,12 @@ static inline void mmap_accounting(size_t size) { } void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) { - if(log_thread_memory_allocations) { - memory_statistics.memory_calls_made++; - memory_statistics.malloc_calls_made++; - memory_statistics.allocated_memory += size; - print_allocations(file, function, line, "malloc()", size); - } + memory_statistics.memory_calls_made++; + memory_statistics.malloc_calls_made++; + memory_statistics.allocated_memory += size; + + if(log_thread_memory_allocations) + log_allocations_int(file, function, line); size_t *n = (size_t *)malloc(sizeof(size_t) + size); if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size); @@ -95,12 +88,11 @@ void *mallocz_int(const char *file, const char *function, const unsigned long li void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) { size = nmemb * size; - if(log_thread_memory_allocations) { - memory_statistics.memory_calls_made++; - memory_statistics.calloc_calls_made++; - memory_statistics.allocated_memory += size; - print_allocations(file, function, line, "calloc()", size); - } + memory_statistics.memory_calls_made++; + memory_statistics.calloc_calls_made++; + memory_statistics.allocated_memory += size; + if(log_thread_memory_allocations) + log_allocations_int(file, function, line); size_t *n = (size_t *)calloc(1, sizeof(size_t) + size); if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size); @@ -118,12 +110,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l n = realloc(n, sizeof(size_t) + size); if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size); - if(log_thread_memory_allocations) { - memory_statistics.memory_calls_made++; - memory_statistics.realloc_calls_made++; - memory_statistics.allocated_memory += (size - old_size); - print_allocations(file, function, line, "realloc()", size - old_size); - } + memory_statistics.memory_calls_made++; + memory_statistics.realloc_calls_made++; + memory_statistics.allocated_memory += (size - old_size); + if(log_thread_memory_allocations) + log_allocations_int(file, function, line); *n = size; return (void *)&n[1]; @@ -132,12 +123,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) { size_t size = strlen(s) + 1; - if(log_thread_memory_allocations) { - memory_statistics.memory_calls_made++; - memory_statistics.strdup_calls_made++; - memory_statistics.allocated_memory += size; - print_allocations(file, function, line, "strdup()", size); - } + memory_statistics.memory_calls_made++; + memory_statistics.strdup_calls_made++; + memory_statistics.allocated_memory += size; + if(log_thread_memory_allocations) + log_allocations_int(file, function, line); size_t *n = (size_t *)malloc(sizeof(size_t) + size); if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size); @@ -155,12 +145,11 @@ void freez_int(const char *file, const char *function, const unsigned long line, n--; size_t size = *n; - if(log_thread_memory_allocations) { - memory_statistics.memory_calls_made++; - memory_statistics.free_calls_made++; - memory_statistics.allocated_memory -= size; - print_allocations(file, function, line, "free()", size); - } + memory_statistics.memory_calls_made++; + memory_statistics.free_calls_made++; + memory_statistics.allocated_memory -= size; + if(log_thread_memory_allocations) + log_allocations_int(file, function, line); free(n); } diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h index 7759438e2d..f937a1bf9c 100644 --- a/libnetdata/libnetdata.h +++ b/libnetdata/libnetdata.h @@ -233,12 +233,15 @@ extern __thread size_t log_thread_memory_allocations; #define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size) #define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size) #define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr) +#define log_allocations() log_allocations_int(__FILE__, __FUNCTION__, __LINE__) extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s); extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size); extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size); extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size); extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr); +extern void log_allocations_int(const char *file, const char *function, const unsigned long line); + #else // NETDATA_LOG_ALLOCATIONS extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL; extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL; diff --git a/libnetdata/procfile/procfile.c b/libnetdata/procfile/procfile.c index ce412f4b08..19964da17d 100644 --- a/libnetdata/procfile/procfile.c +++ b/libnetdata/procfile/procfile.c @@ -4,9 +4,9 @@ #define PF_PREFIX "PROCFILE" -#define PFWORDS_INCREASE_STEP 200 -#define PFLINES_INCREASE_STEP 10 -#define PROCFILE_INCREMENT_BUFFER 512 +#define PFWORDS_INCREASE_STEP 2000 +#define PFLINES_INCREASE_STEP 200 +#define PROCFILE_INCREMENT_BUFFER 4096 int procfile_open_flags = O_RDONLY; @@ -48,9 +48,12 @@ static inline void pfwords_add(procfile *ff, char *str) { pfwords *fw = ff->words; if(unlikely(fw->len == fw->size)) { // debug(D_PROCFILE, PF_PREFIX ": expanding words"); + size_t minimum = PFWORDS_INCREASE_STEP; + size_t optimal = fw->size / 2; + size_t wanted = (optimal > minimum)?optimal:minimum; - ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *)); - fw->size += PFWORDS_INCREASE_STEP; + ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *)); + fw->size += wanted; } fw->words[fw->len++] = str; @@ -90,9 +93,12 @@ static inline size_t *pflines_add(procfile *ff) { pflines *fl = ff->lines; if(unlikely(fl->len == fl->size)) { // debug(D_PROCFILE, PF_PREFIX ": expanding lines"); + size_t minimum = PFLINES_INCREASE_STEP; + size_t optimal = fl->size / 2; + size_t wanted = (optimal > minimum)?optimal:minimum; - ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline)); - fl->size += PFLINES_INCREASE_STEP; + ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline)); + fl->size += wanted; } ffline *ffl = &fl->lines[fl->len++]; @@ -272,9 +278,13 @@ procfile *procfile_readall(procfile *ff) { ssize_t x = ff->size - s; if(unlikely(!x)) { - debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", procfile_filename(ff)); - ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER); - ff->size += PROCFILE_INCREMENT_BUFFER; + size_t minimum = PROCFILE_INCREMENT_BUFFER; + size_t optimal = ff->size / 2; + size_t wanted = (optimal > minimum)?optimal:minimum; + + debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted); + ff = reallocz(ff, sizeof(procfile) + ff->size + wanted); + ff->size += wanted; } debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s)); |