Fix telegram-bot rate limit (#13119)

Co-authored-by: ilyam8 <ilya@netdata.cloud>

2 files changed, 36 insertions, 15 deletions

diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in
index d26f84130e..63a9065cc8 100755
--- a/health/notifications/alarm-notify.sh.in
+++ b/health/notifications/alarm-notify.sh.in
@@ -1339,21 +1339,37 @@ send_telegram() {
 
   if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then
     for chatid in ${chatids}; do
-      # https://core.telegram.org/bots/api#sendmessage
-      httpcode=$(docurl ${disableNotification} \
-        --data-urlencode "parse_mode=HTML" \
-        --data-urlencode "disable_web_page_preview=true" \
-        --data-urlencode "text=${emoji} ${message}" \
-        "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
-
-      if [ "${httpcode}" = "200" ]; then
-        info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
-        sent=$((sent + 1))
-      elif [ "${httpcode}" = "401" ]; then
-        error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
-      else
-        error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
-      fi
+      notify_telegram=1
+      notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0}
+
+      while [ ${notify_telegram} -eq 1 ]; do
+        # https://core.telegram.org/bots/api#sendmessage
+        httpcode=$(docurl ${disableNotification} \
+          --data-urlencode "parse_mode=HTML" \
+          --data-urlencode "disable_web_page_preview=true" \
+          --data-urlencode "text=${emoji} ${message}" \
+          "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}")
+
+        notify_telegram=0
+
+        if [ "${httpcode}" = "200" ]; then
+          info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'"
+          sent=$((sent + 1))
+        elif [ "${httpcode}" = "401" ]; then
+          error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token."
+        elif [ "${httpcode}" = "429" ]; then
+          if [ "$notify_retries" -gt 0 ]; then
+            error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded, retrying after 1s."
+            notify_retries=$((notify_retries - 1))
+            notify_telegram=1
+            sleep 1
+          else
+            error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded."
+          fi
+        else
+          error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}."
+        fi
+      done
     done
 
     [ ${sent} -gt 0 ] && return 0
diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf
index b69c6d538a..52de86645f 100755
--- a/health/notifications/health_alarm_notify.conf
+++ b/health/notifications/health_alarm_notify.conf
@@ -443,6 +443,11 @@ SEND_TELEGRAM="YES"
 # Without it, netdata cannot send telegram messages.
 TELEGRAM_BOT_TOKEN=""
 
+# If an API limit error is returned on sending a message, Netdata will retry this number of times before giving up.
+# Setting the number to 0 makes Netdata do no retries (which is the default).
+# See https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this
+TELEGRAM_RETRIES_ON_LIMIT="0"
+
 # To get your chat ID send the command /getid to telegram bot @myidbot
 # (https://t.me/myidbot). Each user also needs to open a conversation with the
 # bot that will be sending notifications.