/api/v2 part 4 (#14706)

* expose the order of group by

* key renames in json wrapper v2

* added group by context and group by units

* added view_average_values

* fix for view_average_values when percentage is specified

* option group-by-labels is enabling the exposure of all the labels that are used for each of the final grouped dimensions

* when executing group by queries, allocate one dimension data at a time - not all of them

* respect hidden dimensions

* cancel running data query on socket error

* use poll to detect socket errors

* use POLLRDHUP to detect half closed connections

* make sure POLLRDHUP is available

* do not destroy aral-by-size arals

* completed documentation of /api/v2/data.

* moved min, max back to view; updated swagger yaml and json

* default format for /api/v2/data is json2

5 files changed, 103 insertions, 13 deletions

diff --git a/libnetdata/aral/aral.c b/libnetdata/aral/aral.c
index 2a4b0687dd..60fe5e39a7 100644
--- a/libnetdata/aral/aral.c
+++ b/libnetdata/aral/aral.c
@@ -885,10 +885,10 @@ void aral_by_size_release(ARAL *ar) {
             fatal("ARAL BY SIZE: double release detected");
 
         aral_by_size_globals.array[size].refcount--;
-        if(!aral_by_size_globals.array[size].refcount) {
-            aral_destroy(aral_by_size_globals.array[size].ar);
-            aral_by_size_globals.array[size].ar = NULL;
-        }
+//        if(!aral_by_size_globals.array[size].refcount) {
+//            aral_destroy(aral_by_size_globals.array[size].ar);
+//            aral_by_size_globals.array[size].ar = NULL;
+//        }
 
         netdata_spinlock_unlock(&aral_by_size_globals.spinlock);
     }
diff --git a/libnetdata/buffer/buffer.h b/libnetdata/buffer/buffer.h
index 1cf1ec981d..d19aad81e8 100644
--- a/libnetdata/buffer/buffer.h
+++ b/libnetdata/buffer/buffer.h
@@ -3,6 +3,7 @@
 #ifndef NETDATA_WEB_BUFFER_H
 #define NETDATA_WEB_BUFFER_H 1
 
+#include "../string/utf8.h"
 #include "../libnetdata.h"
 
 #define WEB_DATA_LENGTH_INCREASE_STEP 1024
@@ -203,18 +204,56 @@ static inline void buffer_strcat(BUFFER *wb, const char *txt) {
 static inline void buffer_json_strcat(BUFFER *wb, const char *txt) {
     if(unlikely(!txt || !*txt)) return;
 
-    const char *t = txt;
+    const unsigned char *t = (const unsigned char *)txt;
     while(*t) {
-        buffer_need_bytes(wb, 100);
-        char *s = &wb->buffer[wb->len];
-        char *d = s;
-        const char *e = &wb->buffer[wb->size - 1]; // remove 1 to make room for the escape character
+        buffer_need_bytes(wb, 110);
+        unsigned char *s = (unsigned char *)&wb->buffer[wb->len];
+        unsigned char *d = s;
+        const unsigned char *e = (unsigned char *)&wb->buffer[wb->size - 10]; // make room for the max escape sequence
 
         while(*t && d < e) {
-            if(unlikely(*t == '\\' || *t == '\"'))
+#ifdef BUFFER_JSON_ESCAPE_UTF
+            if(unlikely(IS_UTF8_STARTBYTE(*t) && IS_UTF8_BYTE(t[1]))) {
+                // UTF-8 multi-byte encoded character
+
+                // find how big this character is (2-4 bytes)
+                size_t utf_character_size = 2;
+                while(utf_character_size < 4 && t[utf_character_size] && IS_UTF8_BYTE(t[utf_character_size]) && !IS_UTF8_STARTBYTE(t[utf_character_size]))
+                    utf_character_size++;
+
+                uint32_t code_point = 0;
+                for (size_t i = 0; i < utf_character_size; i++) {
+                    code_point <<= 6;
+                    code_point |= (t[i] & 0x3F);
+                }
+
+                t += utf_character_size;
+
+                // encode as \u escape sequence
+                *d++ = '\\';
+                *d++ = 'u';
+                *d++ = hex_digits[(code_point >> 12) & 0xf];
+                *d++ = hex_digits[(code_point >> 8) & 0xf];
+                *d++ = hex_digits[(code_point >> 4) & 0xf];
+                *d++ = hex_digits[code_point & 0xf];
+            }
+            else
+#endif
+            if(unlikely(*t < ' ')) {
+                uint32_t v = *t++;
                 *d++ = '\\';
+                *d++ = 'u';
+                *d++ = hex_digits[(v >> 12) & 0xf];
+                *d++ = hex_digits[(v >> 8) & 0xf];
+                *d++ = hex_digits[(v >> 4) & 0xf];
+                *d++ = hex_digits[v & 0xf];
+            }
+            else {
+                if (unlikely(*t == '\\' || *t == '\"'))
+                    *d++ = '\\';
 
-            *d++ = *t++;
+                *d++ = *t++;
+            }
         }
 
         wb->len += d - s;
diff --git a/libnetdata/socket/socket.c b/libnetdata/socket/socket.c
index eb8e3a93da..dd36da105e 100644
--- a/libnetdata/socket/socket.c
+++ b/libnetdata/socket/socket.c
@@ -1,5 +1,13 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE // for POLLRDHUP
+#endif
+
+#ifndef __BSD_VISIBLE
+#define __BSD_VISIBLE // for POLLRDHUP
+#endif
+
 #include "../libnetdata.h"
 
 // --------------------------------------------------------------------------------------------------------------------
@@ -11,6 +19,46 @@
 #define LARGE_SOCK_SIZE 4096
 #endif
 
+bool fd_is_socket(int fd) {
+    int type;
+    socklen_t len = sizeof(type);
+    if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &type, &len) == -1)
+        return false;
+
+    return true;
+}
+
+bool sock_has_output_error(int fd) {
+    if(fd < 0) {
+        //internal_error(true, "invalid socket %d", fd);
+        return false;
+    }
+
+//    if(!fd_is_socket(fd)) {
+//        //internal_error(true, "fd %d is not a socket", fd);
+//        return false;
+//    }
+
+    short int errors = POLLERR | POLLHUP | POLLNVAL;
+
+#ifdef POLLRDHUP
+    errors |= POLLRDHUP;
+#endif
+
+    struct pollfd pfd = {
+            .fd = fd,
+            .events = POLLOUT | errors,
+            .revents = 0,
+    };
+
+    if(poll(&pfd, 1, 0) == -1) {
+        //internal_error(true, "poll() failed");
+        return false;
+    }
+
+    return ((pfd.revents & errors) || !(pfd.revents & POLLOUT));
+}
+
 int sock_setnonblock(int fd) {
     int flags;
 
diff --git a/libnetdata/socket/socket.h b/libnetdata/socket/socket.h
index 9577453d53..3a8369fba0 100644
--- a/libnetdata/socket/socket.h
+++ b/libnetdata/socket/socket.h
@@ -74,6 +74,9 @@ ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
 ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
 #endif
 
+bool fd_is_socket(int fd);
+bool sock_has_output_error(int fd);
+
 int sock_setnonblock(int fd);
 int sock_delnonblock(int fd);
 int sock_setreuse(int fd, int reuse);
diff --git a/libnetdata/string/utf8.h b/libnetdata/string/utf8.h
index 133ec710b6..3e6c8c2883 100644
--- a/libnetdata/string/utf8.h
+++ b/libnetdata/string/utf8.h
@@ -3,7 +3,7 @@
 #ifndef NETDATA_STRING_UTF8_H
 #define NETDATA_STRING_UTF8_H 1
 
-#define IS_UTF8_BYTE(x) (x & 0x80)
-#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x)&&(x & 0x40))
+#define IS_UTF8_BYTE(x) ((x) & 0x80)
+#define IS_UTF8_STARTBYTE(x) (IS_UTF8_BYTE(x)&&((x) & 0x40))
 
 #endif /* NETDATA_STRING_UTF8_H */