WEBRTC for communication between agents and browsers (#14874)

* initial webrtc setup

* missing files

* rewrite of webrtc integration

* initialization and cleanup of webrtc connections

* make it compile without libdatachannel

* add missing webrtc_initialize() function when webrtc is not enabled

* make c++17 optional

* add build/m4/ax_compiler_vendor.m4

* add ax_cxx_compile_stdcxx.m4

* added new m4 files to makefile.am

* id all webrtc connections

* show warning when webrtc is disabled

* fixed message

* moved all webrtc error checking inside webrtc.cpp

* working webrtc connection establishment and cleanup

* remove obsolete code

* rewrote webrtc code in C to remove dependency for c++17

* fixed left-over reference

* detect binary and text messages

* minor fix

* naming of webrtc threads

* added webrtc configuration

* fix for thread_get_name_np()

* smaller web_client memory footprint

* universal web clients cache

* free web clients every 100 uses

* webrtc is now enabled by default only when compiled with internal checks

* webrtc responses to /api/ requests, including LZ4 compression

* fix for binary and text messages

* web_client_cache is now global

* unification of the internal web server API, for web requests, aclk request, webrtc requests

* more cleanup and unification of web client timings

* fixed compiler warnings

* update sent and received bytes

* eliminated of almost all big buffers in web client

* registry now uses the new json generation

* cookies are now an array; fixed redirects

* fix redirects, again

* write cookies directly to the header buffer, eliminating the need for cookie structures in web client

* reset the has_cookies flag

* gathered all web client cleanup to one function

* fixes redirects

* added summary.globals in /api/v2/data response

* ars to arc in /api/v2/data

* properly handle host impersonation

* set the context of mem.numa_nodes

1 files changed, 22 insertions, 7 deletions

diff --git a/web/server/static/static-threaded.c b/web/server/static/static-threaded.c
index d0d55ebc30..52bb56cd61 100644
--- a/web/server/static/static-threaded.c
+++ b/web/server/static/static-threaded.c
@@ -28,7 +28,7 @@ long web_client_streaming_rate_t = 0L;
 static struct web_client *web_client_create_on_fd(POLLINFO *pi) {
     struct web_client *w;
 
-    w = web_client_get_from_cache_or_allocate();
+    w = web_client_get_from_cache();
     w->ifd = w->ofd = pi->fd;
 
     strncpyz(w->client_ip,   pi->client_ip,   sizeof(w->client_ip) - 1);
@@ -39,7 +39,19 @@ static struct web_client *web_client_create_on_fd(POLLINFO *pi) {
     if(unlikely(!*w->client_port)) strcpy(w->client_port, "-");
 	w->port_acl = pi->port_acl;
 
-    web_client_initialize_connection(w);
+    int flag = 1;
+    if(unlikely(web_client_check_tcp(w) && setsockopt(w->ifd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0))
+        debug(D_WEB_CLIENT, "%llu: failed to enable TCP_NODELAY on socket fd %d.", w->id, w->ifd);
+
+    flag = 1;
+    if(unlikely(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0))
+        debug(D_WEB_CLIENT, "%llu: failed to enable SO_KEEPALIVE on socket fd %d.", w->id, w->ifd);
+
+    web_client_update_acl_matches(w);
+    web_client_enable_wait_receive(w);
+
+    web_server_log_connection(w, "CONNECTED");
+
     w->pollinfo_slot = pi->slot;
     return(w);
 }
@@ -107,7 +119,10 @@ static void web_server_file_del_callback(POLLINFO *pi) {
 
     if(unlikely(!w->pollinfo_slot)) {
         debug(D_WEB_CLIENT, "%llu: CROSS WEB CLIENT CLEANUP (iFD %d, oFD %d)", w->id, pi->fd, w->ofd);
-        web_client_release(w);
+        web_server_log_connection(w, "DISCONNECTED");
+        web_client_request_done(w);
+        web_client_release_to_cache(w);
+        global_statistics_web_client_disconnected();
     }
 
     worker_is_idle();
@@ -278,7 +293,10 @@ static void web_server_del_callback(POLLINFO *pi) {
             pi->flags |= POLLINFO_FLAG_DONT_CLOSE;
 
         debug(D_WEB_CLIENT, "%llu: CLOSING CLIENT FD %d", w->id, pi->fd);
-        web_client_release(w);
+        web_server_log_connection(w, "DISCONNECTED");
+        web_client_request_done(w);
+        web_client_release_to_cache(w);
+        global_statistics_web_client_disconnected();
     }
 
     worker_is_idle();
@@ -402,9 +420,6 @@ cleanup:
 static void socket_listen_main_static_threaded_worker_cleanup(void *ptr) {
     worker_private = (struct web_server_static_threaded_worker *)ptr;
 
-    info("freeing local web clients cache...");
-    web_client_cache_destroy();
-
     info("stopped after %zu connects, %zu disconnects (max concurrent %zu), %zu receptions and %zu sends",
             worker_private->connected,
             worker_private->disconnected,