PCP: support for 'dynamic columns' added at runtime

Implements support for arbitrary Performance Co-Pilot
metrics with per-process instance domains to form new
htop columns.  The column-to-metric mappings are setup
using configuration files which will be documented via
man pages as part of a follow-up commit.

We provide an initial set of column configurations so
as to provide new capabilities to pcp-htop: including
configs for containers, open fd counts, scheduler run
queue time, tcp/udp bytes/calls sent/recv, delay acct,
virtual machine guests, detailed virtual memory, swap.

Note there is a change to the configuration file path
resolution algorithm introduced for 'dynamic meters'.
First, look in any custom PCP_HTOP_DIR location.  Then
iterate, in priority order, users home directory, then
local sysadmins files in /etc/pcp/htop, then readonly
configuration files below /usr/share/pcp/htop.  This
final location becomes the preferred place for our own
shipped meter and column files.

The Settings file (htoprc) writing code is updated to
not using the numeric identifier for dynamic columns.
The same strategy used for dynamic meters is used here
where we write Dynamic(name) so the name can be setup
once more at start.  Regular (static) columns writing
to htoprc - i.e. numerically indexed - is unchanged.

59 files changed, 1063 insertions, 174 deletions

diff --git a/Action.c b/Action.c
index 62cb4525..8a1a47dc 100644
--- a/Action.c
+++ b/Action.c
@@ -13,9 +13,10 @@ in the source distribution for its full text.
 #include <stdbool.h>
 #include <stdlib.h>
 
+#include "CRT.h"
 #include "CategoriesPanel.h"
 #include "CommandScreen.h"
-#include "CRT.h"
+#include "DynamicColumn.h"
 #include "EnvScreen.h"
 #include "FunctionBar.h"
 #include "Hashtable.h"
@@ -168,8 +169,16 @@ static Htop_Reaction actionSetSortColumn(State* st) {
    Panel* sortPanel = Panel_new(0, 0, 0, 0, Class(ListItem), true, FunctionBar_newEnterEsc("Sort   ", "Cancel "));
    Panel_setHeader(sortPanel, "Sort by");
    const ProcessField* fields = st->settings->fields;
+   Hashtable* dynamicColumns = st->settings->dynamicColumns;
    for (int i = 0; fields[i]; i++) {
-      char* name = String_trim(Process_fields[fields[i]].name);
+      char* name = NULL;
+      if (fields[i] >= LAST_PROCESSFIELD) {
+         DynamicColumn* column = Hashtable_get(dynamicColumns, fields[i]);
+         if (column)
+            name = xStrdup(column->caption ? column->caption : column->name);
+      } else {
+         name = String_trim(Process_fields[fields[i]].name);
+      }
       Panel_add(sortPanel, (Object*) ListItem_new(name, fields[i]));
       if (fields[i] == Settings_getActiveSortKey(st->settings))
          Panel_setSelected(sortPanel, i);
diff --git a/AvailableColumnsPanel.c b/AvailableColumnsPanel.c
index 04e4fa58..f6c1868e 100644
--- a/AvailableColumnsPanel.c
+++ b/AvailableColumnsPanel.c
@@ -7,15 +7,20 @@ in the source distribution for its full text.
 
 #include "AvailableColumnsPanel.h"
 
+#include <assert.h>
 #include <ctype.h>
 #include <stdbool.h>
 #include <stdlib.h>
 
 #include "ColumnsPanel.h"
+#include "DynamicColumn.h"
 #include "FunctionBar.h"
+#include "Hashtable.h"
 #include "ListItem.h"
+#include "Macros.h"
 #include "Object.h"
 #include "Process.h"
+#include "ProcessList.h"
 #include "ProvideCurses.h"
 #include "XUtils.h"
 
@@ -29,6 +34,15 @@ static void AvailableColumnsPanel_delete(Object* object) {
    free(this);
 }
 
+static void AvailableColumnsPanel_insert(AvailableColumnsPanel* this, int at, int key) {
+   const char* name;
+   if (key >= LAST_PROCESSFIELD)
+      name = DynamicColumn_init(key);
+   else
+      name = Process_fields[key].name;
+   Panel_insert(this->columns, at, (Object*) ListItem_new(name, key));
+}
+
 static HandlerResult AvailableColumnsPanel_eventHandler(Panel* super, int ch) {
    AvailableColumnsPanel* this = (AvailableColumnsPanel*) super;
    HandlerResult result = IGNORED;
@@ -42,10 +56,9 @@ static HandlerResult AvailableColumnsPanel_eventHandler(Panel* super, int ch) {
          if (!selected)
             break;
 
-         int key = selected->key;
          int at = Panel_getSelectedIndex(this->columns);
-         Panel_insert(this->columns, at, (Object*) ListItem_new(Process_fields[key].name, key));
-         Panel_setSelected(this->columns, at+1);
+         AvailableColumnsPanel_insert(this, at, selected->key);
+         Panel_setSelected(this->columns, at + 1);
          ColumnsPanel_update(this->columns);
          result = HANDLED;
          break;
@@ -68,14 +81,25 @@ const PanelClass AvailableColumnsPanel_class = {
    .eventHandler = AvailableColumnsPanel_eventHandler
 };
 
-AvailableColumnsPanel* AvailableColumnsPanel_new(Panel* columns) {
-   AvailableColumnsPanel* this = AllocThis(AvailableColumnsPanel);
-   Panel* super = (Panel*) this;
-   FunctionBar* fuBar = FunctionBar_new(AvailableColumnsFunctions, NULL, NULL);
-   Panel_init(super, 1, 1, 1, 1, Class(ListItem), true, fuBar);
+static void AvailableColumnsPanel_addDynamicColumn(ht_key_t key, void* value, void* data) {
+   const DynamicColumn* column = (const DynamicColumn*) value;
+   Panel* super = (Panel*) data;
+   const char* title = column->caption ? column->caption : column->heading;
+   if (!title)
+      title = column->name;  // fallback to the only mandatory field
+   char description[256];
+   xSnprintf(description, sizeof(description), "%s - %s", title, column->description);
+   Panel_add(super, (Object*) ListItem_new(description, key));
+}
 
-   Panel_setHeader(super, "Available Columns");
+// Handle DynamicColumns entries in the AvailableColumnsPanel
+static void AvailableColumnsPanel_addDynamicColumns(Panel* super, Hashtable* dynamicColumns) {
+   assert(dynamicColumns);
+   Hashtable_foreach(dynamicColumns, AvailableColumnsPanel_addDynamicColumn, super);
+}
 
+// Handle remaining Platform Meter entries in the AvailableColumnsPanel
+static void AvailableColumnsPanel_addPlatformColumn(Panel* super) {
    for (int i = 1; i < LAST_PROCESSFIELD; i++) {
       if (i != COMM && Process_fields[i].description) {
          char description[256];
@@ -83,6 +107,18 @@ AvailableColumnsPanel* AvailableColumnsPanel_new(Panel* columns) {
          Panel_add(super, (Object*) ListItem_new(description, i));
       }
    }
+}
+
+AvailableColumnsPanel* AvailableColumnsPanel_new(Panel* columns, Hashtable* dynamicColumns) {
+   AvailableColumnsPanel* this = AllocThis(AvailableColumnsPanel);
+   Panel* super = (Panel*) this;
+   FunctionBar* fuBar = FunctionBar_new(AvailableColumnsFunctions, NULL, NULL);
+   Panel_init(super, 1, 1, 1, 1, Class(ListItem), true, fuBar);
+
+   Panel_setHeader(super, "Available Columns");
+   AvailableColumnsPanel_addPlatformColumn(super);
+   AvailableColumnsPanel_addDynamicColumns(super, dynamicColumns);
+
    this->columns = columns;
    return this;
 }
diff --git a/AvailableColumnsPanel.h b/AvailableColumnsPanel.h
index bf87dcfb..99477b6e 100644
--- a/AvailableColumnsPanel.h
+++ b/AvailableColumnsPanel.h
@@ -7,7 +7,9 @@ Released under the GNU GPLv2, see the COPYING file
 in the source distribution for its full text.
 */
 
+#include "Hashtable.h"
 #include "Panel.h"
+#include "ProcessList.h"
 
 
 typedef struct AvailableColumnsPanel_ {
@@ -17,6 +19,6 @@ typedef struct AvailableColumnsPanel_ {
 
 extern const PanelClass AvailableColumnsPanel_class;
 
-AvailableColumnsPanel* AvailableColumnsPanel_new(Panel* columns);
+AvailableColumnsPanel* AvailableColumnsPanel_new(Panel* columns, Hashtable* dynamicColumns);
 
 #endif
diff --git a/AvailableMetersPanel.c b/AvailableMetersPanel.c
index 384c8864..55a45d15 100644
--- a/AvailableMetersPanel.c
+++ b/AvailableMetersPanel.c
@@ -14,8 +14,10 @@ in the source distribution for its full text.
 #include "CPUMeter.h"
 #include "DynamicMeter.h"
 #include "FunctionBar.h"
+#include "Hashtable.h"
 #include "Header.h"
 #include "ListItem.h"
+#include "Macros.h"
 #include "Meter.h"
 #include "MetersPanel.h"
 #include "Object.h"
diff --git a/CategoriesPanel.c b/CategoriesPanel.c
index 4ee1ad46..9adc7be5 100644
--- a/CategoriesPanel.c
+++ b/CategoriesPanel.c
@@ -56,7 +56,7 @@ static void CategoriesPanel_makeColorsPage(CategoriesPanel* this) {
 
 static void CategoriesPanel_makeColumnsPage(CategoriesPanel* this) {
    Panel* columns = (Panel*) ColumnsPanel_new(this->settings);
-   Panel* availableColumns = (Panel*) AvailableColumnsPanel_new(columns);
+   Panel* availableColumns = (Panel*) AvailableColumnsPanel_new(columns, this->settings->dynamicColumns);
    ScreenManager_add(this->scr, columns, 20);
    ScreenManager_add(this->scr, availableColumns, -1);
 }
diff --git a/ColumnsPanel.c b/ColumnsPanel.c
index 5382db0b..c49e8f78 100644
--- a/ColumnsPanel.c
+++ b/ColumnsPanel.c
@@ -11,7 +11,9 @@ in the source distribution for its full text.
 #include <stdlib.h>
 
 #include "CRT.h"
+#include "DynamicColumn.h"
 #include "FunctionBar.h"
+#include "Hashtable.h"
 #include "ListItem.h"
 #include "Object.h"
 #include "Process.h"
@@ -115,6 +117,32 @@ const PanelClass ColumnsPanel_class = {
    .eventHandler = ColumnsPanel_eventHandler
 };
 
+typedef struct {
+   Panel* super;
+   unsigned int id;
+   unsigned int offset;
+} DynamicIterator;
+
+static void ColumnsPanel_add(Panel* super, unsigned int key, Hashtable* columns) {
+   const char* name;
+   if (key < LAST_PROCESSFIELD) {
+      name = Process_fields[key].name;
+   } else {
+      const DynamicColumn* column = Hashtable_get(columns, key);
+      assert(column);
+      if (!column) {
+         name = NULL;
+      } else {
+         name = column->caption ? column->caption : column->heading;
+         if (!name)
+            name = column->name; /* name is a mandatory field */
+      }
+   }
+   if (name == NULL)
+      name = "- ";
+   Panel_add(super, (Object*) ListItem_new(name, key));
+}
+
 ColumnsPanel* ColumnsPanel_new(Settings* settings) {
    ColumnsPanel* this = AllocThis(ColumnsPanel);
    Panel* super = (Panel*) this;
@@ -125,12 +153,11 @@ ColumnsPanel* ColumnsPanel_new(Settings* settings) {
    this->moving = false;
    Panel_setHeader(super, "Active Columns");
 
-   const ProcessField* fields = this->settings->fields;
-   for (; *fields; fields++) {
-      if (Process_fields[*fields].name) {
-         Panel_add(super, (Object*) ListItem_new(Process_fields[*fields].name, *fields));
-      }
-   }
+   Hashtable* dynamicColumns = settings->dynamicColumns;
+   const ProcessField* fields = settings->fields;
+   for (; *fields; fields++)
+      ColumnsPanel_add(super, *fields, dynamicColumns);
+
    return this;
 }
 
@@ -143,7 +170,8 @@ void ColumnsPanel_update(Panel* super) {
    for (int i = 0; i < size; i++) {
       int key = ((ListItem*) Panel_get(super, i))->key;
       this->settings->fields[i] = key;
-      this->settings->flags |= Process_fields[key].flags;
+      if (key < LAST_PROCESSFIELD)
+         this->settings->flags |= Process_fields[key].flags;
    }
    this->settings->fields[size] = 0;
 }
diff --git a/ColumnsPanel.h b/ColumnsPanel.h
index 8bc806eb..39503a50 100644
--- a/ColumnsPanel.h
+++ b/ColumnsPanel.h
@@ -10,6 +10,7 @@ in the source distribution for its full text.
 #include <stdbool.h>
 
 #include "Panel.h"
+#include "ProcessList.h"
 #include "Settings.h"
 
 
diff --git a/CommandLine.c b/CommandLine.c
index d932bb04..b4144d8d 100644
--- a/CommandLine.c
+++ b/CommandLine.c
@@ -22,6 +22,7 @@ in the source distribution for its full text.
 
 #include "Action.h"
 #include "CRT.h"
+#include "DynamicColumn.h"
 #include "DynamicMeter.h"
 #include "Hashtable.h"
 #include "Header.h"
@@ -292,10 +293,14 @@ int CommandLine_run(const char* name, int argc, char** argv) {
    Process_setupColumnWidths();
 
    UsersTable* ut = UsersTable_new();
+   Hashtable* dc = DynamicColumns_new();
    Hashtable* dm = DynamicMeters_new();
-   ProcessList* pl = ProcessList_new(ut, dm, flags.pidMatchList, flags.userId);
+   if (!dc)
+      dc = Hashtable_new(0, true);
 
-   Settings* settings = Settings_new(pl->activeCPUs);
+   ProcessList* pl = ProcessList_new(ut, dm, dc, flags.pidMatchList, flags.userId);
+
+   Settings* settings = Settings_new(pl->activeCPUs, dc);
    pl->settings = settings;
 
    Header* header = Header_new(pl, settings, 2);
@@ -384,8 +389,12 @@ int CommandLine_run(const char* name, int argc, char** argv) {
    if (flags.pidMatchList)
       Hashtable_delete(flags.pidMatchList);
 
-   /* Delete Settings last, since it can get accessed in the crash handler */
+   /* Delete these last, since they can get accessed in the crash handler */
    Settings_delete(settings);
+   if (dc)
+      Hashtable_delete(dc);
+   if (dm)
+      Hashtable_delete(dm);
 
    return 0;
 }
diff --git a/DynamicColumn.c b/DynamicColumn.c
new file mode 100644
index 00000000..62b38238
--- /dev/null
+++ b/DynamicColumn.c
@@ -0,0 +1,60 @@
+/*
+htop - DynamicColumn.c
+(C) 2021 Sohaib Mohammed
+(C) 2021 htop dev team
+(C) 2021 Red Hat, Inc.  All Rights Reserved.
+Released under the GNU GPLv2, see the COPYING file
+in the source distribution for its full text.
+*/
+
+#include "config.h" // IWYU pragma: keep
+
+#include "DynamicColumn.h"
+
+#include <stddef.h>
+
+#include "Platform.h"
+#include "ProcessList.h"
+#include "RichString.h"
+#include "XUtils.h"
+
+
+Hashtable* DynamicColumns_new(void) {
+   return Platform_dynamicColumns();
+}
+
+const char* DynamicColumn_init(unsigned int key) {
+   return Platform_dynamicColumnInit(key);
+}
+
+typedef struct {
+   const char* name;
+   const DynamicColumn* data;
+   unsigned int key;
+} DynamicIterator;
+
+static void DynamicColumn_compare(ht_key_t key, void* value, void* data) {
+   const DynamicColumn* column = (const DynamicColumn*)value;
+   DynamicIterator* iter = (DynamicIterator*)data;
+   if (String_eq(iter->name, column->name)) {
+      iter->data = column;
+      iter->key = key;
+   }
+}
+
+const DynamicColumn* DynamicColumn_search(Hashtable* dynamics, const char* name, unsigned int* key) {
+   DynamicIterator iter = { .key = 0, .data = NULL, .name = name };
+   if (dynamics)
+      Hashtable_foreach(dynamics, DynamicColumn_compare, &iter);
+   if (key)
+      *key = iter.key;
+   return iter.data;
+}
+
+const DynamicColumn* DynamicColumn_lookup(Hashtable* dynamics, unsigned int key) {
+   return (const DynamicColumn*) Hashtable_get(dynamics, key);
+}
+
+bool DynamicColumn_writeField(const Process* proc, RichString* str, unsigned int key) {
+   return Platform_dynamicColumnWriteField(proc, str, key);
+}
diff --git a/DynamicColumn.h b/DynamicColumn.h
new file mode 100644
index 00000000..464ca668
--- /dev/null
+++ b/DynamicColumn.h
@@ -0,0 +1,31 @@
+#ifndef HEADER_DynamicColumn
+#define HEADER_DynamicColumn
+
+#include "Hashtable.h"
+#include "Process.h"
+#include "ProcessList.h"
+#include "RichString.h"
+
+
+#define DYNAMIC_MAX_COLUMN_WIDTH 28
+#define DYNAMIC_DEFAULT_COLUMN_WIDTH -5
+
+typedef struct DynamicColumn_ {
+   char name[32];     /* unique, internal-only name */
+   char* heading;     /* displayed in main screen */
+   char* caption;     /* displayed in setup menu (short name) */
+   char* description; /* displayed in setup menu (detail) */
+   int width;         /* display width +/- for value alignment */
+} DynamicColumn;
+
+Hashtable* DynamicColumns_new(void);
+
+const char* DynamicColumn_init(unsigned int key);
+
+const DynamicColumn* DynamicColumn_lookup(Hashtable* dynamics, unsigned int key);
+
+const DynamicColumn* DynamicColumn_search(Hashtable* dynamics, const char* name, unsigned int* field);
+
+bool DynamicColumn_writeField(const Process* proc, RichString* str, unsigned int key);
+
+#endif
diff --git a/DynamicMeter.h b/DynamicMeter.h
index 582bc277..f787cbb7 100644
--- a/DynamicMeter.h
+++ b/DynamicMeter.h
@@ -1,6 +1,8 @@
 #ifndef HEADER_DynamicMeter
 #define HEADER_DynamicMeter
 
+#include <stdbool.h>
+
 #include "Hashtable.h"
 #include "Meter.h"
 
@@ -11,8 +13,6 @@ typedef struct DynamicMeter_ {
    char* description;
    unsigned int type;
    double maximum;
-
-   void* dynamicData;  /* platform-specific meter data */
 } DynamicMeter;
 
 Hashtable* DynamicMeters_new(void);
diff --git a/Makefile.am b/Makefile.am
index 1e027c6a..fc19e7c5 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -39,6 +39,7 @@ myhtopsources = \
 	DateTimeMeter.c \
 	DiskIOMeter.c \
 	DisplayOptionsPanel.c \
+	DynamicColumn.c \
 	DynamicMeter.c \
 	EnvScreen.c \
 	FunctionBar.c \
@@ -94,6 +95,7 @@ myhtopheaders = \
 	DateTimeMeter.h \
 	DiskIOMeter.h \
 	DisplayOptionsPanel.h \
+	DynamicColumn.h \
 	DynamicMeter.h \
 	EnvScreen.h \
 	FunctionBar.h \
@@ -359,25 +361,27 @@ endif
 # --------------------------
 
 pcp_platform_headers = \
+	linux/PressureStallMeter.h \
+	linux/ZramMeter.h \
+	linux/ZramStats.h \
+	pcp/PCPDynamicColumn.h \
 	pcp/PCPDynamicMeter.h \
 	pcp/PCPProcess.h \
 	pcp/PCPProcessList.h \
 	pcp/Platform.h \
 	pcp/ProcessField.h \
-	linux/PressureStallMeter.h \
-	linux/ZramMeter.h \
-	linux/ZramStats.h \
 	zfs/ZfsArcMeter.h \
 	zfs/ZfsArcStats.h \
 	zfs/ZfsCompressedArcMeter.h
 
 pcp_platform_sources = \
+	linux/PressureStallMeter.c \
+	linux/ZramMeter.c \
+	pcp/PCPDynamicColumn.c \
 	pcp/PCPDynamicMeter.c \
 	pcp/PCPProcess.c \
 	pcp/PCPProcessList.c \
 	pcp/Platform.c \
-	linux/PressureStallMeter.c \
-	linux/ZramMeter.c \
 	zfs/ZfsArcMeter.c \
 	zfs/ZfsCompressedArcMeter.c
 
diff --git a/Process.c b/Process.c
index dfb3f7ba..fbc9f732 100644
--- a/Process.c
+++ b/Process.c
@@ -26,6 +26,7 @@ in the source distribution for its full text.
 #include "Macros.h"
 #include "Platform.h"
 #include "ProcessList.h"
+#include "DynamicColumn.h"
 #include "RichString.h"
 #include "Settings.h"
 #include "XUtils.h"
@@ -905,8 +906,11 @@ void Process_writeField(const Process* this, RichString* str, ProcessField field
       xSnprintf(buffer, n, "%-9d ", this->st_uid);
       break;
    default:
+      if (DynamicColumn_writeField(this, str, field))
+         return;
       assert(0 && "Process_writeField: default key reached"); /* should never be reached */
       xSnprintf(buffer, n, "- ");
+      break;
    }
    RichString_appendAscii(str, attr, buffer);
 }
diff --git a/Process.h b/Process.h
index 6e69f9c3..c3166927 100644
--- a/Process.h
+++ b/Process.h
@@ -56,6 +56,7 @@ typedef enum ProcessField_ {
    /* Platform specific fields, defined in ${platform}/ProcessField.h */
    PLATFORM_PROCESS_FIELDS
 
+   /* Do not add new fields after this entry (dynamic entries follow) */
    LAST_PROCESSFIELD
 } ProcessField;
 
@@ -267,7 +268,7 @@ typedef struct ProcessFieldData_ {
    /* Scan flag to enable scan-method otherwise not run */
    uint32_t flags;
 
-   /* Whether the values are process identifies; adjusts the width of title and values if true */
+   /* Whether the values are process identifiers; adjusts the width of title and values if true */
    bool pidColumn;
 
    /* Whether the column should be sorted in descending order by default */
diff --git a/ProcessList.c b/ProcessList.c
index 8ab4d8f6..daf871a7 100644
--- a/ProcessList.c
+++ b/ProcessList.c
@@ -12,6 +12,7 @@ in the source distribution for its full text.
 #include <string.h>
 
 #include "CRT.h"
+#include "DynamicColumn.h"
 #include "Hashtable.h"
 #include "Macros.h"
 #include "Platform.h"
@@ -19,7 +20,7 @@ in the source distribution for its full text.
 #include "XUtils.h"
 
 
-ProcessList* ProcessList_init(ProcessList* this, const ObjectClass* klass, UsersTable* usersTable, Hashtable* dynamicMeters, Hashtable* pidMatchList, uid_t userId) {
+ProcessList* ProcessList_init(ProcessList* this, const ObjectClass* klass, UsersTable* usersTable, Hashtable* dynamicMeters, Hashtable* dynamicColumns, Hashtable* pidMatchList, uid_t userId) {
    this->processes = Vector_new(klass, true, DEFAULT_SIZE);
    this->processes2 = Vector_new(klass, true, DEFAULT_SIZE); // tree-view auxiliary buffer
 
@@ -30,6 +31,7 @@ ProcessList* ProcessList_init(ProcessList* this, const ObjectClass* klass, Users
    this->usersTable = usersTable;
    this->pidMatchList = pidMatchList;
    this->dynamicMeters = dynamicMeters;
+   this->dynamicColumns = dynamicColumns;
 
    this->userId = userId;
 
@@ -83,7 +85,22 @@ void ProcessList_setPanel(ProcessList* this, Panel* panel) {
    this->panel = panel;
 }
 
-static const char* alignedProcessFieldTitle(ProcessField field) {
+static const char* alignedDynamicColumnTitle(const ProcessList* this, int key) {
+   const DynamicColumn* column = Hashtable_get(this->dynamicColumns, key);
+   if (column == NULL)
+      return "- ";
+   static char titleBuffer[DYNAMIC_MAX_COLUMN_WIDTH + /* space */ 1 + /* null terminator */ + 1];
+   int width = column->width;
+   if (!width || abs(width) > DYNAMIC_MAX_COLUMN_WIDTH)
+      width = DYNAMIC_DEFAULT_COLUMN_WIDTH;
+   xSnprintf(titleBuffer, sizeof(titleBuffer), "%*s", width, column->heading);
+   return titleBuffer;
+}
+
+static const char* alignedProcessFieldTitle(const ProcessList* this, ProcessField field) {