Merge pull request #3160 from kattunga/master

add support for alerta.io 

2 files changed, 123 insertions, 2 deletions

diff --git a/conf.d/health_alarm_notify.conf b/conf.d/health_alarm_notify.conf
index eb01e2bb96..86dbb217ce 100644..100755
--- a/conf.d/health_alarm_notify.conf
+++ b/conf.d/health_alarm_notify.conf
@@ -7,6 +7,7 @@
 # - e-mails (using the sendmail command),
 # - push notifications to your mobile phone (pushover.net),
 # - messages to your slack team (slack.com),
+# - messages to your alerta server (alerta.io),
 # - messages to your flock team (flock.com),
 # - messages to your discord guild (discordapp.com),
 # - messages to your telegram chat / group chat (telegram.org)
@@ -23,7 +24,7 @@
 #------------------------------------------------------------------------------
 # proxy configuration
 #
-# If you need to send curl based notifications (pushover, pushbullet, slack,
+# If you need to send curl based notifications (pushover, pushbullet, slack, alerta,
 # flock, discord, telegram) via a proxy, set these to your proxy address:
 #export http_proxy="http://10.0.0.1:3128/"
 #export https_proxy="http://10.0.0.1:3128/"
@@ -64,6 +65,7 @@ curl=""
 #  - pushover user tokens
 #  - telegram chat ids
 #  - slack channels
+#  - alerta environment
 #  - flock rooms
 #  - discord channels
 #  - hipchat rooms
@@ -79,6 +81,7 @@ curl=""
 #  pushover   : "2987343...9437837 8756278...2362736|critical"
 #  telegram   : "111827421 112746832|critical"
 #  slack      : "alarms disasters|critical"
+#  alerta     : "alarms disasters|critical"
 #  flock      : "alarms disasters|critical"
 #  discord    : "alarms disasters|critical"
 #  twilio     : "+15555555555 +17777777777|critical"
@@ -87,7 +90,7 @@ curl=""
 #  pd         : "<pd_service_key_1> <pd_service_key_2>|critical"
 #
 # If a recipient is set to empty string, the default recipient of the given
-# notification method (email, pushover, telegram, slack, etc) will be used.
+# notification method (email, pushover, telegram, slack, alerta, etc) will be used.
 # To disable a notification, use the recipient called: disabled
 # This works for all notification methods (including the default recipients).
 
@@ -276,6 +279,31 @@ DEFAULT_RECIPIENT_SLACK=""
 
 
 #------------------------------------------------------------------------------
+# alerta (alerta.io) global notification options
+
+# multiple recipients (Environments) can be given like this:
+#                  "Production Development ..."
+
+# enable/disable sending alerta notifications
+SEND_ALERTA="YES"
+
+# set your alerta server API url
+# this is the API url you define when install Alerta server, it is the same for
+# all users.
+ALERTA_WEBHOOK_URL=""
+
+# Login with an administrative user to you Alerta server and create an API KEY
+# with write permisions.
+ALERTA_API_KEY=""
+
+# you can define environments in /etc/alertad.conf option ALLOWED_ENVIRONMENTS
+# standard environments are Production and Development
+# if a role's recipients are not configured, a notification will be send to
+# this Environment (empty = do not send a notification for unconfigured roles):
+DEFAULT_RECIPIENT_ALERTA=""
+
+
+#------------------------------------------------------------------------------
 # flock (flock.com) global notification options
 
 # enable/disable sending flock notifications
@@ -442,6 +470,8 @@ role_recipients_telegram[sysadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[sysadmin]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_alerta[sysadmin]="${DEFAULT_RECIPIENT_ALERTA}"
+
 role_recipients_flock[sysadmin]="${DEFAULT_RECIPIENT_FLOCK}"
 
 role_recipients_discord[sysadmin]="${DEFAULT_RECIPIENT_DISCORD}"
@@ -471,6 +501,8 @@ role_recipients_telegram[domainadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[domainadmin]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_alerta[domainadmin]="${DEFAULT_RECIPIENT_ALERTA}"
+
 role_recipients_flock[domainadmin]="${DEFAULT_RECIPIENT_FLOCK}"
 
 role_recipients_discord[domainadmin]="${DEFAULT_RECIPIENT_DISCORD}"
@@ -501,6 +533,8 @@ role_recipients_telegram[dba]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[dba]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_alerta[dba]="${DEFAULT_RECIPIENT_ALERTA}"
+
 role_recipients_flock[dba]="${DEFAULT_RECIPIENT_FLOCK}"
 
 role_recipients_discord[dba]="${DEFAULT_RECIPIENT_DISCORD}"
@@ -531,6 +565,8 @@ role_recipients_telegram[webmaster]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[webmaster]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_alerta[webmaster]="${DEFAULT_RECIPIENT_ALERTA}"
+
 role_recipients_flock[webmaster]="${DEFAULT_RECIPIENT_FLOCK}"
 
 role_recipients_discord[webmaster]="${DEFAULT_RECIPIENT_DISCORD}"
@@ -561,6 +597,8 @@ role_recipients_telegram[proxyadmin]="${DEFAULT_RECIPIENT_TELEGRAM}"
 
 role_recipients_slack[proxyadmin]="${DEFAULT_RECIPIENT_SLACK}"
 
+role_recipients_alerta[proxyadmin]="${DEFAULT_RECIPIENT_ALERTA}"
+
 role_recipients_flock[proxyadmin]="${DEFAULT_RECIPIENT_FLOCK}"
 
 role_recipients_discord[proxyadmin]="${DEFAULT_RECIPIENT_DISCORD}"
diff --git a/plugins.d/alarm-notify.sh b/plugins.d/alarm-notify.sh
index 48b24b1278..a124da0442 100755
--- a/plugins.d/alarm-notify.sh
+++ b/plugins.d/alarm-notify.sh
@@ -16,6 +16,7 @@
 # Supported notification methods:
 #  - emails by @ktsaou
 #  - slack.com notifications by @ktsaou
+#  - alerta.io notifications by @kattunga
 #  - discordapp.com notifications by @lowfive
 #  - pushover.net notifications by @ktsaou
 #  - pushbullet.com push notifications by Tiago Peralta @tperalta82 #1070
@@ -219,6 +220,7 @@ sendmail=
 
 # enable / disable features
 SEND_SLACK="YES"
+SEND_ALERTA="YES"
 SEND_FLOCK="YES"
 SEND_DISCORD="YES"
 SEND_PUSHOVER="YES"
@@ -238,6 +240,12 @@ SLACK_WEBHOOK_URL=
 DEFAULT_RECIPIENT_SLACK=
 declare -A role_recipients_slack=()
 
+# alerta configs
+ALERTA_WEBHOOK_URL=
+ALERTA_API_KEY=
+DEFAULT_RECIPIENT_ALERTA=
+declare -A role_recipients_alerta=()
+
 # flock configs
 FLOCK_WEBHOOK_URL=
 DEFAULT_RECIPIENT_FLOCK=
@@ -386,6 +394,7 @@ filter_recipient_by_criticality() {
 # find the recipients' addresses per method
 
 declare -A arr_slack=()
+declare -A arr_alerta=()
 declare -A arr_flock=()
 declare -A arr_discord=()
 declare -A arr_pushover=()
@@ -479,6 +488,14 @@ do
         [ "${r}" != "disabled" ] && filter_recipient_by_criticality slack "${r}" && arr_slack[${r/|*/}]="1"
     done
 
+    # alerta
+    a="${role_recipients_alerta[${x}]}"
+    [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_ALERTA}"
+    for r in ${a//,/ }
+    do
+        [ "${r}" != "disabled" ] && filter_recipient_by_criticality alerta "${r}" && arr_alerta[${r/|*/}]="1"
+    done
+
     # flock
     a="${role_recipients_flock[${x}]}"
     [ -z "${a}" ] && a="${DEFAULT_RECIPIENT_FLOCK}"
@@ -517,6 +534,10 @@ done
 to_slack="${!arr_slack[*]}"
 [ -z "${to_slack}" ] && SEND_SLACK="NO"
 
+# build the list of alerta recipients (channels)
+to_alerta="${!arr_alerta[*]}"
+[ -z "${to_alerta}" ] && SEND_ALERTA="NO"
+
 # build the list of flock recipients (channels)
 to_flock="${!arr_flock[*]}"
 [ -z "${to_flock}" ] && SEND_FLOCK="NO"
@@ -577,6 +598,9 @@ done
 # check slack
 [ -z "${SLACK_WEBHOOK_URL}" ] && SEND_SLACK="NO"
 
+# check alerta
+[ -z "${ALERTA_WEBHOOK_URL}" ] && SEND_ALERTA="NO"
+
 # check flock
 [ -z "${FLOCK_WEBHOOK_URL}" ] && SEND_FLOCK="NO"
 
@@ -624,6 +648,7 @@ fi
 if [ \( \
            "${SEND_PUSHOVER}"    = "YES" \
         -o "${SEND_SLACK}"       = "YES" \
+        -o "${SEND_ALERTA}"      = "YES" \
         -o "${SEND_FLOCK}"       = "YES" \
         -o "${SEND_DISCORD}"     = "YES" \
         -o "${SEND_HIPCHAT}"     = "YES" \
@@ -644,6 +669,7 @@ if [ \( \
         SEND_PUSHBULLET="NO"
         SEND_TELEGRAM="NO"
         SEND_SLACK="NO"
+        SEND_ALERTA="NO"
         SEND_FLOCK="NO"
         SEND_DISCORD="NO"
         SEND_TWILIO="NO"
@@ -671,6 +697,7 @@ if [   "${SEND_EMAIL}"          != "YES" \
     -a "${SEND_PUSHOVER}"       != "YES" \
     -a "${SEND_TELEGRAM}"       != "YES" \
     -a "${SEND_SLACK}"          != "YES" \
+    -a "${SEND_ALERTA}"         != "YES" \
     -a "${SEND_FLOCK}"          != "YES" \
     -a "${SEND_DISCORD}"        != "YES" \
     -a "${SEND_TWILIO}"         != "YES" \
@@ -1252,6 +1279,52 @@ EOF
 }
 
 # -----------------------------------------------------------------------------
+# alerta sender
+
+send_alerta() {
+    local webhook="${1}" channels="${2}" httpcode sent=0 channel severity content
+
+    [ "${SEND_ALERTA}" != "YES" ] && return 1
+
+    case "${status}" in
+        WARNING)  severity="warning" ;;
+        CRITICAL) severity="critical" ;;
+        CLEAR)    severity="cleared" ;;
+        *)        severity="unknown" ;;
+    esac
+
+    info=$( echo -n ${info})
+
+    for channel in ${channels}
+    do
+        content="{"
+        content="$content \"environment\": \"${channel}\","
+        content="$content \"service\": [\"${host}\"],"
+        content="$content \"resource\": \"${host}\","
+        content="$content \"event\": \"${chart}.${name}\","
+        content="$content \"severity\": \"${severity}\","
+        content="$content \"value\": \"${alarm}\","
+        content="$content \"text\": \"${info}\""
+        content="$content }"
+
+
+        httpcode=$(docurl -X POST "${webhook}" -H "Content-Type: application/json" -H "Authorization: Key $ALERTA_API_KEY" -d "$content" )
+
+        if [[ "${httpcode}" = "200" || "${httpcode}" = "201" ]]
+        then
+            info "sent alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}'"
+            sent=$((sent + 1))
+        else
+            error "failed to send alerta notification for: ${host} ${chart}.${name} is ${status} to '${channel}', with HTTP error code ${httpcode}."
+        fi
+    done
+
+    [ ${sent} -gt 0 ] && return 0
+
+    return 1
+}
+
+# -----------------------------------------------------------------------------
 # flock sender
 
 send_flock() {
@@ -1470,6 +1543,15 @@ send_slack "${SLACK_WEBHOOK_URL}" "${to_slack}"
 SENT_SLACK=$?
 
 # -----------------------------------------------------------------------------
+# send the alerta notification
+
+# alerta aggregates posts from the same username
+# so we use "${host} ${status}" as the bot username, to make them diff
+
+send_alerta "${ALERTA_WEBHOOK_URL}" "${to_alerta}"
+SENT_ALERTA=$?
+
+# -----------------------------------------------------------------------------
 # send the flock notification
 
 # flock aggregates posts from the same username
@@ -1737,6 +1819,7 @@ if [   ${SENT_EMAIL}        -eq 0 \
     -o ${SENT_PUSHOVER}     -eq 0 \
     -o ${SENT_TELEGRAM}     -eq 0 \
     -o ${SENT_SLACK}        -eq 0 \
+    -o ${SENT_ALERTA}       -eq 0 \
     -o ${SENT_FLOCK}        -eq 0 \
     -o ${SENT_DISCORD}      -eq 0 \
     -o ${SENT_TWILIO}       -eq 0 \