Added libwebsockets bundling code to netdata-installer.sh (#8144)

* Generalize handling of fetch and verification of archives.

This generalizes the logic used for fetching and verifying source
archives for external dependencies that we potentially bundle as part of
our regular build.

It also slightly improves the error handling for the bundling of
libmosquitto.

* Add code to bundle libwebsockets during install.

This gets used when building on a system without libwebsockets3.

Due to how the libwebsockets build system works, this adds cmake as an
installer dependency on systems that do not have libwebsockets3.

* Add libwebsockets-dev to install-required-packages.sh

* Fix error caused by resolving merge conflict incorrectly.

* Unconditionally bunlde libwebsockets if cloud is enabled on install.

As discussed with the product team, we're taking this approach to ensure
SOCKS5 proxy support.

5 files changed, 61 insertions, 4 deletions

diff --git a/packaging/installer/functions.sh b/packaging/installer/functions.sh
index 4fe911ce68..fd4b3550f4 100644
--- a/packaging/installer/functions.sh
+++ b/packaging/installer/functions.sh
@@ -104,6 +104,37 @@ get() {
 }
 
 # -----------------------------------------------------------------------------
+# external component handling
+
+fetch_and_verify() {
+  local component=${1}
+  local url=${2}
+  local base_name=${3}
+  local tmp=${4}
+  local override=${5}
+
+  if [ -z "${override}" ]; then
+     get "${url}" > "${tmp}/${base_name}"
+  else
+    progress "Using provided ${component} archive ${override}"
+    run cp "${override}" "${tmp}/${base_name}"
+  fi
+
+  if [ ! -f "${tmp}/${base_name}" ] || [ ! -s "${tmp}/${base_name}" ]; then
+    run_failed "Unable to find usable archive for ${component}"
+    return 1
+  fi
+
+  grep "${base_name}\$" "${INSTALLER_DIR}/packaging/${component}.checksums" > "${tmp}/sha256sums.txt" 2> /dev/null
+
+  # Checksum validation
+  if ! (cd "${tmp}" && safe_sha256sum -c "sha256sums.txt"); then
+    run_failed "${component} files checksum validation failed."
+    return 1
+  fi
+}
+
+# -----------------------------------------------------------------------------
 
 netdata_banner() {
   local l1="  ^" \
diff --git a/packaging/installer/install-required-packages.sh b/packaging/installer/install-required-packages.sh
index b562f9ebed..6876bd2f57 100755
--- a/packaging/installer/install-required-packages.sh
+++ b/packaging/installer/install-required-packages.sh
@@ -591,6 +591,12 @@ declare -A pkg_automake=(
   ['default']="automake"
 )
 
+# Required to build libwebsockets and libmosquitto on some systems.
+declare -A pkg_cmake=(
+  ['clearlinux']="c-basic"
+  ['default']="cmake"
+)
+
 declare -A pkg_bridge_utils=(
   ['gentoo']="net-misc/bridge-utils"
   ['clearlinux']="network-basic"
@@ -703,6 +709,12 @@ declare -A pkg_libmnl_dev=(
   ['default']=""
 )
 
+declare -A pkg_libwebsockets_dev=(
+  ['debian']="libwebsockets-dev"
+  ['ubuntu']="libwebsockets-dev"
+  ['default']="libwebsockets-devel"
+)
+
 declare -A pkg_lm_sensors=(
   ['alpine']="lm_sensors"
   ['arch']="lm_sensors"
@@ -1104,6 +1116,7 @@ packages() {
   require_cmd autogen || suitable_package autogen
   require_cmd automake || suitable_package automake
   require_cmd pkg-config || suitable_package pkg-config
+  require_cmd cmake || suitable_package cmake
 
   # -------------------------------------------------------------------------
   # debugging tools for development
@@ -1157,6 +1170,7 @@ packages() {
     suitable_package libz-dev
     suitable_package libuuid-dev
     suitable_package libmnl-dev
+    suitable_package libwebsockets-dev
   fi
 
   # -------------------------------------------------------------------------
diff --git a/packaging/installer/methods/manual.md b/packaging/installer/methods/manual.md
index 2ce2a6c2e9..802aeea99c 100644
--- a/packaging/installer/methods/manual.md
+++ b/packaging/installer/methods/manual.md
@@ -61,16 +61,16 @@ This is how to do it by hand:
 
 ```sh
 # Debian / Ubuntu
-apt-get install zlib1g-dev uuid-dev libuv1-dev liblz4-dev libjudy-dev libssl-dev libmnl-dev gcc make git autoconf autoconf-archive autogen automake pkg-config curl python
+apt-get install zlib1g-dev uuid-dev libuv1-dev liblz4-dev libjudy-dev libssl-dev libmnl-dev gcc make git autoconf autoconf-archive autogen automake pkg-config curl python cmake libwebsockets-dev
 
 # Fedora
-dnf install zlib-devel libuuid-devel libuv-devel lz4-devel Judy-devel openssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python
+dnf install zlib-devel libuuid-devel libuv-devel lz4-devel Judy-devel openssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python cmake libwebsockets-devel
 
 # CentOS / Red Hat Enterprise Linux
-yum install autoconf automake curl gcc git libmnl-devel libuuid-devel openssl-devel libuv-devel lz4-devel Judy-devel make nc pkgconfig python zlib-devel
+yum install autoconf automake curl gcc git libmnl-devel libuuid-devel openssl-devel libuv-devel lz4-devel Judy-devel make nc pkgconfig python zlib-devel cmake libwebsockets-devel
 
 # openSUSE
-zypper install zlib-devel libuuid-devel libuv-devel liblz4-devel judy-devel libopenssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python
+zypper install zlib-devel libuuid-devel libuv-devel liblz4-devel judy-devel libopenssl-devel libmnl-devel gcc make git autoconf autoconf-archive autogen automake pkgconfig curl findutils python cmake libwebsockets-devel
 ```
 
 Once Netdata is compiled, to run it the following packages are required (already installed using the above commands):
@@ -115,6 +115,16 @@ Netdata DB engine can be enabled when these are installed (they are optional):
 
 *Netdata will greatly benefit if you have the above packages installed, but it will still work without them.*
 
+Netdata Cloud support may require the following packages to be installed:
+
+| package  | description
+|:--------:| -----------------------
+| `libwebsockets-devel` | Version 3 or higher is at needed build time for websockets support for Netdata Cloud
+| `libwebsockets` | Version 3 or higher is needed at runtime for websockets support for Netdata Cloud
+| `cmake` | Needed at build time if you aren't using your distribution's version of libwebsockets or are building on a platform other than Linux
+
+*Netdata will greatly benefit if you have the above packages installed, but it will still work without them.*
+
 ### CentOS / RHEL 6.x
 
 On CentOS / RHEL 6.x, many of the dependencies for Netdata are only
diff --git a/packaging/libwebsockets.checksums b/packaging/libwebsockets.checksums
new file mode 100644
index 0000000000..fc9167c0fa
--- /dev/null
+++ b/packaging/libwebsockets.checksums
@@ -0,0 +1 @@
+166d6e17cab64bfc10c2a71799c298284540a1fa63f6ea3de5caccb34502243c  v3.2.2.tar.gz
diff --git a/packaging/libwebsockets.version b/packaging/libwebsockets.version
new file mode 100644
index 0000000000..be94e6f53d
--- /dev/null
+++ b/packaging/libwebsockets.version
@@ -0,0 +1 @@
+3.2.2