Health Alarm to Dynatrace Event implementation (#8476)

* Health Alarm to Dynatrace Event implementation

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Patti Short <35278231+shortpatti@users.noreply.github.com>

* Removing unwanted "

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Update health/notifications/dynatrace/README.md

Co-Authored-By: Joel Hans <joel.g.hans@gmail.com>

* Implementation of https://github.com/netdata/netdata/pull/8476/checks?check_run_id=533657899

* Implemented https://github.com/netdata/netdata/pull/8476/checks?check_run_id=534740578

Co-authored-by: Patti Short <35278231+shortpatti@users.noreply.github.com>
Co-authored-by: Joel Hans <joel.g.hans@gmail.com>

4 files changed, 170 insertions, 4 deletions

diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in
index 8f0e14a27d..64dc83d008 100755
--- a/health/notifications/alarm-notify.sh.in
+++ b/health/notifications/alarm-notify.sh.in
@@ -35,6 +35,7 @@
 #  - Microsoft Team notification by @tioumen
 #  - RocketChat notifications by @Hermsi1337 #3777
 #  - Google Hangouts Chat notifications by @EnzoAkira and @hendrikhofstadt
+#  - Dynatrace Event by @illumine
 
 # -----------------------------------------------------------------------------
 # testing notifications
@@ -170,6 +171,7 @@ awssns
 rocketchat
 sms
 hangouts
+dynatrace
 "
 
 # -----------------------------------------------------------------------------
@@ -366,6 +368,16 @@ IRC_NETWORK=
 # hangouts configs
 declare -A HANGOUTS_WEBHOOK_URI
 
+# dynatrace configs
+DYNATRACE_SPACE=
+DYNATRACE_SERVER=
+DYNATRACE_TOKEN=
+DYNATRACE_TAG_VALUE=
+DYNATRACE_ANNOTATION_TYPE=
+DYNATRACE_EVENT=
+SEND_DYNATRACE=
+
+
 # load the stock and user configuration files
 # these will overwrite the variables above
 
@@ -503,6 +515,14 @@ filter_recipient_by_criticality() {
 #shellcheck disable=SC2153
 { [ -z "${FLEEP_SERVER}" ] || [ -z "${FLEEP_SENDER}" ]; } && SEND_FLEEP="NO"
 
+# check dynatrace
+{ [ -z "${DYNATRACE_SPACE}" ] ||
+  [ -z "${DYNATRACE_SERVER}" ] ||
+  [ -z "${DYNATRACE_TOKEN}" ] ||
+  [ -z "${DYNATRACE_TAG_VALUE}" ] ||
+  [ -z "${DYNATRACE_EVENT}" ]; } && SEND_DYNATRACE="NO"
+
+
 if [ "${SEND_PUSHOVER}" = "YES" ] ||
 	[ "${SEND_SLACK}" = "YES" ] ||
 	[ "${SEND_ROCKETCHAT}" = "YES" ] ||
@@ -521,7 +541,8 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
 	[ "${SEND_PROWL}" = "YES" ] ||
 	[ "${SEND_HANGOUTS}" = "YES" ] ||
 	[ "${SEND_CUSTOM}" = "YES" ] ||
-	[ "${SEND_MSTEAM}" = "YES" ]; then
+	[ "${SEND_MSTEAM}" = "YES" ] ||
+	[ "${SEND_DYNATRACE}" = "YES" ]; then
 	# if we need curl, check for the curl command
 	if [ -z "${curl}" ]; then
 		curl="$(command -v curl 2>/dev/null)"
@@ -547,6 +568,7 @@ if [ "${SEND_PUSHOVER}" = "YES" ] ||
 		SEND_PROWL="NO"
 		SEND_HANGOUTS="NO"
 		SEND_CUSTOM="NO"
+		SEND_DYNATRACE="NO"
 	fi
 fi
 
@@ -673,7 +695,9 @@ for method in "${SEND_EMAIL}" \
 	"${SEND_AWSSNS}" \
 	"${SEND_SYSLOG}" \
 	"${SEND_SMS}" \
-	"${SEND_MSTEAM}"; do
+	"${SEND_MSTEAM}" \
+    "${SEND_DYNATRACE}"; do
+	
 	if [ "${method}" == "YES" ]; then
 		proceed=1
 		break
@@ -1887,6 +1911,51 @@ EOF
 
 	return 1
 }
+# -----------------------------------------------------------------------------
+# Dynatrace sender
+send_dynatrace() {
+  local dynatrace_url="${DYNATRACE_SERVER}/e/${DYNATRACE_SPACE}/api/v1/events"
+  local description="NetData Notification for: ${host} ${chart}.${name} is ${status}"
+  local payload=""
+  
+  payload=$(cat <<EOF
+{
+  "title": "NetData Alarm from ${host}",
+  "source" : "${DYNATRACE_ANNOTATION_TYPE}",
+  "description" : "${description}",
+  "eventType": "${DYNATRACE_EVENT}",
+   "attachRules":{
+     "tagRule":[{
+        "meTypes":["HOST"],
+        "tags":["${DYNATRACE_TAG_VALUE}"]
+     }]
+  },
+  "customProperties":{
+    "description": "${description}"
+  }
+}
+EOF
+)
+
+  # echo ${payload}
+
+  httpcode=$(docurl -X POST  -H "Authorization: Api-token ${DYNATRACE_TOKEN}"  -H "Content-Type: application/json" -d "${payload}"    ${dynatrace_url})
+  ret=$?
+
+
+  if [ ${ret} -eq 0 ]; then
+     if [ "${httpcode}" = "200" ]; then
+        info "sent ${DYNATRACE_EVENT} to ${DYNATRACE_SERVER}"
+        return 0
+     else
+        warning "Dynatrace ${DYNATRACE_SERVER} responded ${httpcode} notification for: ${host} ${chart}.${name} is ${status} was not sent!"
+        return 1
+     fi
+  else
+     error  "failed to sent ${DYNATRACE_EVENT} notification for: ${host} ${chart}.${name} is ${status} to  ${DYNATRACE_SERVER} with error code ${ret}."
+     return 1
+  fi
+}
 
 # -----------------------------------------------------------------------------
 # prepare the content of the notification
@@ -2392,6 +2461,12 @@ fi
 SENT_EMAIL=$?
 
 # -----------------------------------------------------------------------------
+# send the EVENT to Dynatrace
+send_dynatrace "${host}" "${chart}" "${name}"  "${status}"
+SENT_DYNATRACE=$?
+
+
+# -----------------------------------------------------------------------------
 # let netdata know
 for state in "${SENT_EMAIL}" \
 	"${SENT_PUSHOVER}" \
@@ -2416,7 +2491,8 @@ for state in "${SENT_EMAIL}" \
 	"${SENT_AWSSNS}" \
 	"${SENT_SYSLOG}" \
 	"${SENT_SMS}" \
-	"${SENT_MSTEAM}"; do
+	"${SENT_MSTEAM}" \
+    "${SENT_DYNATRACE}"; do	
 	if [ "${state}" -eq 0 ]; then
 		# we sent something
 		exit 0
diff --git a/health/notifications/dynatrace/Makefile.inc b/health/notifications/dynatrace/Makefile.inc
new file mode 100644
index 0000000000..a2ae623fbc
--- /dev/null
+++ b/health/notifications/dynatrace/Makefile.inc
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# THIS IS NOT A COMPLETE Makefile
+# IT IS INCLUDED BY ITS PARENT'S Makefile.am
+# IT IS REQUIRED TO REFERENCE ALL FILES RELATIVE TO THE PARENT
+
+# install these files
+dist_noinst_DATA += \
+    dynatrace/README.md \
+    dynatrace/Makefile.inc \
+    $(NULL)
+
diff --git a/health/notifications/dynatrace/README.md b/health/notifications/dynatrace/README.md
new file mode 100644
index 0000000000..3532968f05
--- /dev/null
+++ b/health/notifications/dynatrace/README.md
@@ -0,0 +1,36 @@
+<!--
+---
+title: "Dynatrace"
+custom_edit_url: https://github.com/netdata/netdata/edit/master/health/notifications/dynatrace/README.md
+---
+-->
+
+# Dynatrace
+
+Dynatrace allows you to receive notifications using their Events REST API.
+
+See [the Dynatrace documentation](https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/environment-api/events/post-event/) about POSTing an event in the Events API for more details.
+
+
+
+You need:
+
+1.  Dynatrace Server. You can use the same on all your Netdata servers but make sure the server is network visible from your Netdata hosts.
+The Dynatrace server should be with protocol prefixed (`http://` or `https://`). For example: `https://monitor.example.com`
+This is a required parameter.
+2.  API Token. Generate a secure access API token that enables access to your Dynatrace monitoring data via the REST-based API.
+Generate a Dynatrace API authentication token. On your Dynatrace server, go to **Settings** --> **Integration** --> **Dynatrace API** --> **Generate token**.
+See [Dynatrace API - Authentication](https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/basics/dynatrace-api-authentication/) for more details.
+This is a required parameter.
+3.  API Space. This is the URL part of the page you have access in order to generate the API Token. For example, for my generated API Token the URL is:
+https://monitor.illumineit.com/e/2a93fe0e-4cd5-469a-9d0d-1a064235cfce/#settings/integration/apikeys;gf=all
+In that case, my space is _2a93fe0e-4cd5-469a-9d0d-1a064235cfce_
+This is a required parameter.
+4. Generate a Server Tag. On your Dynatrace Server, go to **Settings** --> **Tags** --> **Manually applied tags** and create the Tag.
+The Netdata alarm is sent as a Dynatrace Event to be correlated with all those hosts tagged with this Tag you have created.
+This is a required parameter.
+5. Specify the Dynatrace event. This can be one of `CUSTOM_INFO`, `CUSTOM_ANNOTATION`, `CUSTOM_CONFIGURATION`, and `CUSTOM_DEPLOYMENT`. 
+The default value is `CUSTOM_INFO`.
+This is a required parameter.
+6. Specify the annotation type. This is the source of the Dynatrace event. Put whatever it fits you, for example, 
+_Netdata Alarm_, which is also the default value.
diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf
index 5540dfc134..7baffb68b1 100755
--- a/health/notifications/health_alarm_notify.conf
+++ b/health/notifications/health_alarm_notify.conf
@@ -230,6 +230,43 @@ DEFAULT_RECIPIENT_EMAIL="root"
 #EMAIL_PLAINTEXT_ONLY="YES"
 
 #------------------------------------------------------------------------------
+# Dynatrace global notification options
+#------------------------------------------------------------------------------
+# enable/disable sending Dynatrace notifications
+SEND_DYNATRACE="YES"
+
+# The Dynatrace server with protocol prefix (http:// or https://), example https://monitor.illumineit.com
+# Required
+DYNATRACE_SERVER=""
+
+# Generate a Dynatrace API authentication token
+# Read https://www.dynatrace.com/support/help/extend-dynatrace/dynatrace-api/basics/dynatrace-api-authentication/ 
+# On Dynatrace server goto Settings --> Integration --> Dynatrace API --> Generate token
+# Required
+DYNATRACE_TOKEN=""
+
+# Beware: Space is taken from dynatrace URL from browser when you create the TOKEN
+# Required
+DYNATRACE_SPACE=""
+
+# Generate a Server Tag. On the Dynatrace Server go to Settings --> Tags --> Manually applied tags create the Tag
+# The NetData alarm will be sent as a Dynatrace Event to be correlated with all those hosts tagged with this Tag 
+# you created.
+# Required
+DYNATRACE_TAG_VALUE=""
+
+# Change this to what you want
+DYNATRACE_ANNOTATION_TYPE="NetData Alarm"
+
+# This can be CUSTOM_INFO, CUSTOM_ANNOTATION, CUSTOM_CONFIGURATION, CUSTOM_DEPLOYMENT
+# Applying default value
+# Required
+DYNATRACE_EVENT="CUSTOM_INFO"
+
+
+DEFAULT_RECIPIENT_DYNATRACE=""
+
+#------------------------------------------------------------------------------
 # hangouts (google hangouts chat) global notification options
 
 # enable/disable sending hangouts notifications
@@ -873,6 +910,7 @@ role_recipients_msteam[sysadmin]="${DEFAULT_RECIPIENT_MSTEAM}"
 
 role_recipients_rocketchat[sysadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
 
+role_recipients_dynatrace[sysadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
 # -----------------------------------------------------------------------------
 # DNS related alarms
 
@@ -922,6 +960,7 @@ role_recipients_rocketchat[domainadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
 
 role_recipients_sms[domainadmin]="${DEFAULT_RECIPIENT_SMS}"
 
+role_recipients_dynatrace[domainadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
 # -----------------------------------------------------------------------------
 # database servers alarms
 # mysql, redis, memcached, postgres, etc
@@ -972,6 +1011,7 @@ role_recipients_rocketchat[dba]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
 
 role_recipients_sms[dba]="${DEFAULT_RECIPIENT_SMS}"
 
+role_recipients_dynatrace[dba]="${DEFAULT_RECIPIENT_DYNATRACE}"
 # -----------------------------------------------------------------------------
 # web servers alarms
 # apache, nginx, lighttpd, etc
@@ -1022,6 +1062,7 @@ role_recipients_rocketchat[webmaster]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
 
 role_recipients_sms[webmaster]="${DEFAULT_RECIPIENT_SMS}"
 
+role_recipients_dynatrace[webmaster]="${DEFAULT_RECIPIENT_DYNATRACE}"
 # -----------------------------------------------------------------------------
 # proxy servers alarms
 # squid, etc
@@ -1072,7 +1113,7 @@ role_recipients_rocketchat[proxyadmin]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
 
 role_recipients_sms[proxyadmin]="${DEFAULT_RECIPIENT_SMS}"
 
-
+role_recipients_dynatrace[proxyadmin]="${DEFAULT_RECIPIENT_DYNATRACE}"
 # -----------------------------------------------------------------------------
 # peripheral devices
 # UPS, photovoltaics, etc
@@ -1121,3 +1162,4 @@ role_recipients_rocketchat[sitemgr]="${DEFAULT_RECIPIENT_ROCKETCHAT}"
 
 role_recipients_sms[sitemgr]="${DEFAULT_RECIPIENT_SMS}"
 
+role_recipients_dynatrace[sitemgr]="${DEFAULT_RECIPIENT_DYNATRACE}"
\ No newline at end of file