diff options
author | МАН69К <MAH69IK@users.noreply.github.com> | 2022-08-19 19:21:56 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-19 19:21:56 +0300 |
commit | 1cbd7ab42d8eedd12dc0f14d29f24e4c84dc3cac (patch) | |
tree | ea21a0dc47f3d8ba2c2288f88fbc9e873481cd1a /health | |
parent | 2df6667153cfcd4b116589315063d73b8ed9b111 (diff) |
Fix telegram-bot rate limit (#13119)
Co-authored-by: ilyam8 <ilya@netdata.cloud>
Diffstat (limited to 'health')
-rwxr-xr-x | health/notifications/alarm-notify.sh.in | 46 | ||||
-rwxr-xr-x | health/notifications/health_alarm_notify.conf | 5 |
2 files changed, 36 insertions, 15 deletions
diff --git a/health/notifications/alarm-notify.sh.in b/health/notifications/alarm-notify.sh.in index d26f84130e..63a9065cc8 100755 --- a/health/notifications/alarm-notify.sh.in +++ b/health/notifications/alarm-notify.sh.in @@ -1339,21 +1339,37 @@ send_telegram() { if [ "${SEND_TELEGRAM}" = "YES" ] && [ -n "${bottoken}" ] && [ -n "${chatids}" ] && [ -n "${message}" ]; then for chatid in ${chatids}; do - # https://core.telegram.org/bots/api#sendmessage - httpcode=$(docurl ${disableNotification} \ - --data-urlencode "parse_mode=HTML" \ - --data-urlencode "disable_web_page_preview=true" \ - --data-urlencode "text=${emoji} ${message}" \ - "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}") - - if [ "${httpcode}" = "200" ]; then - info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'" - sent=$((sent + 1)) - elif [ "${httpcode}" = "401" ]; then - error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token." - else - error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}." - fi + notify_telegram=1 + notify_retries=${TELEGRAM_RETRIES_ON_LIMIT:-0} + + while [ ${notify_telegram} -eq 1 ]; do + # https://core.telegram.org/bots/api#sendmessage + httpcode=$(docurl ${disableNotification} \ + --data-urlencode "parse_mode=HTML" \ + --data-urlencode "disable_web_page_preview=true" \ + --data-urlencode "text=${emoji} ${message}" \ + "https://api.telegram.org/bot${bottoken}/sendMessage?chat_id=${chatid}") + + notify_telegram=0 + + if [ "${httpcode}" = "200" ]; then + info "sent telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}'" + sent=$((sent + 1)) + elif [ "${httpcode}" = "401" ]; then + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': Wrong bot token." + elif [ "${httpcode}" = "429" ]; then + if [ "$notify_retries" -gt 0 ]; then + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded, retrying after 1s." + notify_retries=$((notify_retries - 1)) + notify_telegram=1 + sleep 1 + else + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}': rate limit exceeded." + fi + else + error "failed to send telegram notification for: ${host} ${chart}.${name} is ${status} to '${chatid}' with HTTP response status code ${httpcode}." + fi + done done [ ${sent} -gt 0 ] && return 0 diff --git a/health/notifications/health_alarm_notify.conf b/health/notifications/health_alarm_notify.conf index b69c6d538a..52de86645f 100755 --- a/health/notifications/health_alarm_notify.conf +++ b/health/notifications/health_alarm_notify.conf @@ -443,6 +443,11 @@ SEND_TELEGRAM="YES" # Without it, netdata cannot send telegram messages. TELEGRAM_BOT_TOKEN="" +# If an API limit error is returned on sending a message, Netdata will retry this number of times before giving up. +# Setting the number to 0 makes Netdata do no retries (which is the default). +# See https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this +TELEGRAM_RETRIES_ON_LIMIT="0" + # To get your chat ID send the command /getid to telegram bot @myidbot # (https://t.me/myidbot). Each user also needs to open a conversation with the # bot that will be sending notifications. |