Merging the feature branch for the ACLK in the previous sprint. (#8179)

* ACLK connection and protocol improvements (#8139)
* Adding ACLK retry on connection failure (#8147)
* Fixed reconnect issues on the ACLK. (#8163)
* Cleaning up ACLK - part 1 (#8167)

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

8 files changed, 133 insertions, 51 deletions

diff --git a/build_external/bin/clean-install.sh b/build_external/bin/clean-install.sh
index 032cb09ca4..20bcae66ae 100755
--- a/build_external/bin/clean-install.sh
+++ b/build_external/bin/clean-install.sh
@@ -42,6 +42,11 @@ else
             --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
             --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
         ;;
+        arch-extras)   # Add valgrind to the container
+            docker build -f "$BuildBase/clean-install-arch-extras.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
+            --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
+            --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
+        ;;
         *)
             echo "Unknown $DISTRO-$VERSION"
         ;;
diff --git a/build_external/clean-install-arch-extras.Dockerfile b/build_external/clean-install-arch-extras.Dockerfile
new file mode 100644
index 0000000000..270e74fb0c
--- /dev/null
+++ b/build_external/clean-install-arch-extras.Dockerfile
@@ -0,0 +1,56 @@
+FROM archlinux/base:latest
+
+# There is some redundancy between this file and the archlinux Dockerfile in the helper images
+# repo and also with the clean-install.Dockefile. Once the help image is availabled on Docker
+# Hub this file can be deleted.
+
+RUN pacman -Syyu --noconfirm
+RUN pacman --noconfirm --needed -S autoconf \
+                                   autoconf-archive \
+                                   autogen \
+                                   automake \
+                                   gcc \
+                                   make \
+                                   git \
+                                   libuv \
+                                   lz4 \
+                                   netcat \
+                                   openssl \
+                                   pkgconfig \
+                                   python \
+                                   libvirt \
+                                   libwebsockets \
+                                   valgrind
+
+ARG ACLK=no
+ARG EXTRA_CFLAGS
+COPY . /opt/netdata/source
+WORKDIR /opt/netdata/source
+
+RUN git config --global user.email "root@container"
+RUN git config --global user.name "Fake root"
+
+# RUN make distclean   -> not safe if tree state changed on host since last config
+# Kill everything that is not in .gitignore preserving any fresh changes, i.e. untracked changes will be
+# deleted but local changes to tracked files will be preserved.
+RUN if git status --porcelain | grep '^[MADRC]'; then \
+        git stash && git clean -dxf && (git stash apply || true) \
+    else \
+        git clean -dxf ; \
+    fi
+
+# Not everybody is updating distclean properly - fix.
+RUN find . -name '*.Po' -exec rm \{\} \;
+RUN rm -rf autom4te.cache
+RUN rm -rf .git/
+RUN find . -type f >/opt/netdata/manifest
+
+RUN CFLAGS="-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
+    -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --disable-lto
+
+RUN ln -sf /dev/stdout /var/log/netdata/access.log
+RUN ln -sf /dev/stdout /var/log/netdata/debug.log
+RUN ln -sf /dev/stderr /var/log/netdata/error.log
+
+CMD ["/usr/sbin/valgrind", "--leak-check=full", "/usr/sbin/netdata", "-D"]
+   
diff --git a/build_external/clean-install-arch.Dockerfile b/build_external/clean-install-arch.Dockerfile
index 7725ae3052..2438a56082 100644
--- a/build_external/clean-install-arch.Dockerfile
+++ b/build_external/clean-install-arch.Dockerfile
@@ -50,3 +50,5 @@ RUN CFLAGS="-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -D
 RUN ln -sf /dev/stdout /var/log/netdata/access.log
 RUN ln -sf /dev/stdout /var/log/netdata/debug.log
 RUN ln -sf /dev/stderr /var/log/netdata/error.log
+
+CMD ["/usr/sbin/netdata", "-D"]
\ No newline at end of file
diff --git a/build_external/projects/aclk-testing/agent-valgrind-compose.yml b/build_external/projects/aclk-testing/agent-valgrind-compose.yml
new file mode 100644
index 0000000000..98075156d7
--- /dev/null
+++ b/build_external/projects/aclk-testing/agent-valgrind-compose.yml
@@ -0,0 +1,19 @@
+version: '3.3'
+services:
+  agent_master:
+    build:
+      context: ../../..
+      dockerfile: build_external/make-install.Dockerfile
+      args:
+      - DISTRO=arch
+      - VERSION=extras
+    image: arch_current_dev:latest
+    command: >
+      sh -c "echo -n 00000000-0000-0000-0000-000000000000 >/etc/netdata/claim.d/claimed_id &&
+             echo '[agent_cloud_link]' >>/etc/netdata/netdata.conf &&
+             echo '  agent cloud link hostname = vernemq' >>/etc/netdata/netdata.conf &&
+             echo '  agent cloud link port = 9002' >>/etc/netdata/netdata.conf &&
+             /usr/sbin/valgrind --leak-check=full /usr/sbin/netdata -D"
+    ports:
+    - 20000:19999
+
diff --git a/build_external/projects/aclk-testing/docker-compose.yml b/build_external/projects/aclk-testing/docker-compose.yml
deleted file mode 100644
index 06cf53bb82..0000000000
--- a/build_external/projects/aclk-testing/docker-compose.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-version: '3.3'
-services:
-  agent_master:
-    build:
-      context: ../../..
-      dockerfile: build_external/make_install_fedora_30.Dockerfile
-      args:
-        ACLK: "yes"
-    image: fedora_30_dev
-    command: >
-      sh -c "echo 00000000-0000-0000-0000-000000000000 >/etc/netdata/claim.d/claimed_id &&
-             echo '[agent_cloud_link]' >>/etc/netdata/netdata.conf &&
-             echo '  agent cloud link hostname = 172.22.0.100' >>/etc/netdata/netdata.conf &&
-             echo '  agent cloud link port = 8080' >>/etc/netdata/netdata.conf &&
-             /usr/sbin/netdata -D"
-    ports:
-    - 20000:19999
-    networks:
-      service1_net:
-        ipv4_address: 172.22.0.99
-    #volumes:
-    #- ./master_stream.conf:/etc/netdata/stream.conf:ro
-#  agent_slave1:
-#    image: debian_buster_dev
-#    command: /usr/sbin/netdata -D
-#    ports:
-#    - 20001:19999
-#    volumes:
-#      - ./slave_stream.conf:/etc/netdata/stream.conf:ro
-#  agent_slave2:
-#    image: ubuntu_2004_dev
-#    command: /usr/sbin/netdata -D
-#    ports:
-#    - 20002:19999
-#    volumes:
-#      - ./slave_stream.conf:/etc/netdata/stream.conf:ro
-  vernemq:
-    build:
-      dockerfile: configureVerneMQ.Dockerfile
-      context: .
-    networks:
-      service1_net:
-        ipv4_address: 172.22.0.100
-
-networks:
-  service1_net:
-    ipam:
-      driver: default
-      config:
-        - subnet: 172.22.0.0/16
-
diff --git a/build_external/projects/aclk-testing/paho-compose.yml b/build_external/projects/aclk-testing/paho-compose.yml
new file mode 100644
index 0000000000..4fc6ce2c49
--- /dev/null
+++ b/build_external/projects/aclk-testing/paho-compose.yml
@@ -0,0 +1,6 @@
+version: '3.3'
+services:
+  paho_inspect:
+    build:
+      context: .
+      dockerfile: paho.Dockerfile
\ No newline at end of file
diff --git a/build_external/projects/aclk-testing/paho-inspection.py b/build_external/projects/aclk-testing/paho-inspection.py
new file mode 100644
index 0000000000..ec1e1675b3
--- /dev/null
+++ b/build_external/projects/aclk-testing/paho-inspection.py
@@ -0,0 +1,33 @@
+import ssl
+import paho.mqtt.client as mqtt
+
+def on_connect(mqttc, obj, flags, rc):
+    print("connected rc: "+str(rc), flush=True)
+    mqttc.subscribe("/agent/#",0)
+def on_disconnect(mqttc, obj, flags, rc):
+    print("disconnected rc: "+str(rc), flush=True)
+def on_message(mqttc, obj, msg):
+    print(msg.topic+" "+str(msg.qos)+" "+str(msg.payload), flush=True)
+def on_publish(mqttc, obj, mid):
+    print("mid: "+str(mid), flush=True)
+def on_subscribe(mqttc, obj, mid, granted_qos):
+    print("Subscribed: "+str(mid)+" "+str(granted_qos), flush=True)
+def on_log(mqttc, obj, level, string):
+    print(string)
+print("Starting paho-inspection", flush=True)
+mqttc = mqtt.Client(transport='websockets')
+#mqttc.tls_set(certfile="server.crt", keyfile="server.key", cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
+#mqttc.tls_set(ca_certs="server.crt", cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
+mqttc.tls_set(cert_reqs=ssl.CERT_NONE, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
+mqttc.tls_insecure_set(True)
+mqttc.on_message = on_message
+mqttc.on_connect = on_connect
+mqttc.on_disconnect = on_disconnect
+mqttc.on_publish = on_publish
+mqttc.on_subscribe = on_subscribe
+mqttc.connect("vernemq", 9002, 60)
+
+#mqttc.publish("/agent/mine","Test1")
+#mqttc.subscribe("$SYS/#", 0)
+print("Connected succesfully, monitoring /agent/#", flush=True)
+mqttc.loop_forever()
diff --git a/build_external/projects/aclk-testing/paho.Dockerfile b/build_external/projects/aclk-testing/paho.Dockerfile
new file mode 100644
index 0000000000..77a49e7683
--- /dev/null
+++ b/build_external/projects/aclk-testing/paho.Dockerfile
@@ -0,0 +1,12 @@
+FROM archlinux/base:latest
+
+RUN pacman -Syyu --noconfirm 
+RUN pacman --noconfirm --needed -S python-pip
+
+RUN pip install paho-mqtt
+
+RUN mkdir -p /opt/paho
+COPY paho-inspection.py /opt/paho/
+
+WORKDIR /opt/paho
+CMD ["/usr/sbin/python", "paho-inspection.py"]
\ No newline at end of file