procfile: more comfortable initial settings and faster/fewer reallocs (#12791)

3 files changed, 53 insertions, 51 deletions

diff --git a/libnetdata/libnetdata.c b/libnetdata/libnetdata.c
index 36787042ef..4c38a3a5e7 100644
--- a/libnetdata/libnetdata.c
+++ b/libnetdata/libnetdata.c
@@ -31,6 +31,8 @@ const char *program_version = VERSION;
 // routines.
 
 #ifdef NETDATA_LOG_ALLOCATIONS
+#warning NETDATA_LOG_ALLOCATIONS ENABLED - set log_thread_memory_allocations=1 on any thread to log all its allocations - or use log_allocations() to log them on demand
+
 static __thread struct memory_statistics {
     volatile ssize_t malloc_calls_made;
     volatile ssize_t calloc_calls_made;
@@ -44,23 +46,14 @@ static __thread struct memory_statistics {
 
 __thread size_t log_thread_memory_allocations = 0;
 
-static inline void print_allocations(const char *file, const char *function, const unsigned long line, const char *type, size_t size) {
+inline void log_allocations_int(const char *file, const char *function, const unsigned long line) {
     static __thread struct memory_statistics old = { 0, 0, 0, 0, 0, 0, 0, 0 };
 
-    fprintf(stderr, "%s iteration %zu MEMORY TRACE: %lu@%s : %s : %s : %zu\n",
-            netdata_thread_tag(),
-            log_thread_memory_allocations,
-            line, file, function,
-            type, size
-    );
-
-    fprintf(stderr, "%s iteration %zu MEMORY ALLOCATIONS: (%04lu@%-40.40s:%-40.40s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): %s : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
+    fprintf(stderr, "%s MEMORY ALLOCATIONS: (%04lu@%s:%s): Allocated %zd KiB (%+zd B), mmapped %zd KiB (%+zd B): : malloc %zd (%+zd), calloc %zd (%+zd), realloc %zd (%+zd), strdup %zd (%+zd), free %zd (%+zd)\n",
             netdata_thread_tag(),
-            log_thread_memory_allocations,
             line, file, function,
             (memory_statistics.allocated_memory + 512) / 1024, memory_statistics.allocated_memory - old.allocated_memory,
             (memory_statistics.mmapped_memory + 512) / 1024, memory_statistics.mmapped_memory - old.mmapped_memory,
-            type,
             memory_statistics.malloc_calls_made, memory_statistics.malloc_calls_made - old.malloc_calls_made,
             memory_statistics.calloc_calls_made, memory_statistics.calloc_calls_made - old.calloc_calls_made,
             memory_statistics.realloc_calls_made, memory_statistics.realloc_calls_made - old.realloc_calls_made,
@@ -79,12 +72,12 @@ static inline void mmap_accounting(size_t size) {
 }
 
 void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size) {
-    if(log_thread_memory_allocations) {
-        memory_statistics.memory_calls_made++;
-        memory_statistics.malloc_calls_made++;
-        memory_statistics.allocated_memory += size;
-        print_allocations(file, function, line, "malloc()", size);
-    }
+    memory_statistics.memory_calls_made++;
+    memory_statistics.malloc_calls_made++;
+    memory_statistics.allocated_memory += size;
+
+    if(log_thread_memory_allocations)
+        log_allocations_int(file, function, line);
 
     size_t *n = (size_t *)malloc(sizeof(size_t) + size);
     if (unlikely(!n)) fatal("mallocz() cannot allocate %zu bytes of memory.", size);
@@ -95,12 +88,11 @@ void *mallocz_int(const char *file, const char *function, const unsigned long li
 void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size) {
     size = nmemb * size;
 
-    if(log_thread_memory_allocations) {
-        memory_statistics.memory_calls_made++;
-        memory_statistics.calloc_calls_made++;
-        memory_statistics.allocated_memory += size;
-        print_allocations(file, function, line, "calloc()", size);
-    }
+    memory_statistics.memory_calls_made++;
+    memory_statistics.calloc_calls_made++;
+    memory_statistics.allocated_memory += size;
+    if(log_thread_memory_allocations)
+        log_allocations_int(file, function, line);
 
     size_t *n = (size_t *)calloc(1, sizeof(size_t) + size);
     if (unlikely(!n)) fatal("callocz() cannot allocate %zu bytes of memory.", size);
@@ -118,12 +110,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l
     n = realloc(n, sizeof(size_t) + size);
     if (unlikely(!n)) fatal("reallocz() cannot allocate %zu bytes of memory (from %zu bytes).", size, old_size);
 
-    if(log_thread_memory_allocations) {
-        memory_statistics.memory_calls_made++;
-        memory_statistics.realloc_calls_made++;
-        memory_statistics.allocated_memory += (size - old_size);
-        print_allocations(file, function, line, "realloc()", size - old_size);
-    }
+    memory_statistics.memory_calls_made++;
+    memory_statistics.realloc_calls_made++;
+    memory_statistics.allocated_memory += (size - old_size);
+    if(log_thread_memory_allocations)
+        log_allocations_int(file, function, line);
 
     *n = size;
     return (void *)&n[1];
@@ -132,12 +123,11 @@ void *reallocz_int(const char *file, const char *function, const unsigned long l
 char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s) {
     size_t size = strlen(s) + 1;
 
-    if(log_thread_memory_allocations) {
-        memory_statistics.memory_calls_made++;
-        memory_statistics.strdup_calls_made++;
-        memory_statistics.allocated_memory += size;
-        print_allocations(file, function, line, "strdup()", size);
-    }
+    memory_statistics.memory_calls_made++;
+    memory_statistics.strdup_calls_made++;
+    memory_statistics.allocated_memory += size;
+    if(log_thread_memory_allocations)
+        log_allocations_int(file, function, line);
 
     size_t *n = (size_t *)malloc(sizeof(size_t) + size);
     if (unlikely(!n)) fatal("strdupz() cannot allocate %zu bytes of memory.", size);
@@ -155,12 +145,11 @@ void freez_int(const char *file, const char *function, const unsigned long line,
     n--;
     size_t size = *n;
 
-    if(log_thread_memory_allocations) {
-        memory_statistics.memory_calls_made++;
-        memory_statistics.free_calls_made++;
-        memory_statistics.allocated_memory -= size;
-        print_allocations(file, function, line, "free()", size);
-    }
+    memory_statistics.memory_calls_made++;
+    memory_statistics.free_calls_made++;
+    memory_statistics.allocated_memory -= size;
+    if(log_thread_memory_allocations)
+        log_allocations_int(file, function, line);
 
     free(n);
 }
diff --git a/libnetdata/libnetdata.h b/libnetdata/libnetdata.h
index 7759438e2d..f937a1bf9c 100644
--- a/libnetdata/libnetdata.h
+++ b/libnetdata/libnetdata.h
@@ -233,12 +233,15 @@ extern __thread size_t log_thread_memory_allocations;
 #define mallocz(size) mallocz_int(__FILE__, __FUNCTION__, __LINE__, size)
 #define reallocz(ptr, size) reallocz_int(__FILE__, __FUNCTION__, __LINE__, ptr, size)
 #define freez(ptr) freez_int(__FILE__, __FUNCTION__, __LINE__, ptr)
+#define log_allocations() log_allocations_int(__FILE__, __FUNCTION__, __LINE__)
 
 extern char *strdupz_int(const char *file, const char *function, const unsigned long line, const char *s);
 extern void *callocz_int(const char *file, const char *function, const unsigned long line, size_t nmemb, size_t size);
 extern void *mallocz_int(const char *file, const char *function, const unsigned long line, size_t size);
 extern void *reallocz_int(const char *file, const char *function, const unsigned long line, void *ptr, size_t size);
 extern void freez_int(const char *file, const char *function, const unsigned long line, void *ptr);
+extern void log_allocations_int(const char *file, const char *function, const unsigned long line);
+
 #else // NETDATA_LOG_ALLOCATIONS
 extern char *strdupz(const char *s) MALLOCLIKE NEVERNULL;
 extern void *callocz(size_t nmemb, size_t size) MALLOCLIKE NEVERNULL;
diff --git a/libnetdata/procfile/procfile.c b/libnetdata/procfile/procfile.c
index ce412f4b08..19964da17d 100644
--- a/libnetdata/procfile/procfile.c
+++ b/libnetdata/procfile/procfile.c
@@ -4,9 +4,9 @@
 
 #define PF_PREFIX "PROCFILE"
 
-#define PFWORDS_INCREASE_STEP 200
-#define PFLINES_INCREASE_STEP 10
-#define PROCFILE_INCREMENT_BUFFER 512
+#define PFWORDS_INCREASE_STEP 2000
+#define PFLINES_INCREASE_STEP 200
+#define PROCFILE_INCREMENT_BUFFER 4096
 
 int procfile_open_flags = O_RDONLY;
 
@@ -48,9 +48,12 @@ static inline void pfwords_add(procfile *ff, char *str) {
     pfwords *fw = ff->words;
     if(unlikely(fw->len == fw->size)) {
         // debug(D_PROCFILE, PF_PREFIX ":   expanding words");
+        size_t minimum = PFWORDS_INCREASE_STEP;
+        size_t optimal = fw->size / 2;
+        size_t wanted = (optimal > minimum)?optimal:minimum;
 
-        ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + PFWORDS_INCREASE_STEP) * sizeof(char *));
-        fw->size += PFWORDS_INCREASE_STEP;
+        ff->words = fw = reallocz(fw, sizeof(pfwords) + (fw->size + wanted) * sizeof(char *));
+        fw->size += wanted;
     }
 
     fw->words[fw->len++] = str;
@@ -90,9 +93,12 @@ static inline size_t *pflines_add(procfile *ff) {
     pflines *fl = ff->lines;
     if(unlikely(fl->len == fl->size)) {
         // debug(D_PROCFILE, PF_PREFIX ":   expanding lines");
+        size_t minimum = PFLINES_INCREASE_STEP;
+        size_t optimal = fl->size / 2;
+        size_t wanted = (optimal > minimum)?optimal:minimum;
 
-        ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + PFLINES_INCREASE_STEP) * sizeof(ffline));
-        fl->size += PFLINES_INCREASE_STEP;
+        ff->lines = fl = reallocz(fl, sizeof(pflines) + (fl->size + wanted) * sizeof(ffline));
+        fl->size += wanted;
     }
 
     ffline *ffl = &fl->lines[fl->len++];
@@ -272,9 +278,13 @@ procfile *procfile_readall(procfile *ff) {
         ssize_t x = ff->size - s;
 
         if(unlikely(!x)) {
-            debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s'.", procfile_filename(ff));
-            ff = reallocz(ff, sizeof(procfile) + ff->size + PROCFILE_INCREMENT_BUFFER);
-            ff->size += PROCFILE_INCREMENT_BUFFER;
+            size_t minimum = PROCFILE_INCREMENT_BUFFER;
+            size_t optimal = ff->size / 2;
+            size_t wanted = (optimal > minimum)?optimal:minimum;
+
+            debug(D_PROCFILE, PF_PREFIX ": Expanding data buffer for file '%s' by %zu bytes.", procfile_filename(ff), wanted);
+            ff = reallocz(ff, sizeof(procfile) + ff->size + wanted);
+            ff->size += wanted;
         }
 
         debug(D_PROCFILE, "Reading file '%s', from position %zd with length %zd", procfile_filename(ff), s, (ssize_t)(ff->size - s));