diff options
| author | James Mills <prologic@shortcircuit.net.au> | 2020-05-26 15:58:50 +1000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-05-26 15:58:50 +1000 |
| commit | cea8a3fcbb3b48fce545d19bce2b5cf920c6794d (patch) | |
| tree | 79583fb4ac32a9a60664a7b0b0bebd9addd805b5 | |
| parent | a20e8f163fde7d64cbc258b6903230dda651adf9 (diff) | |
Introduce a random sleep in the Netdata updater (#9079)
* Introduce a random sleep in the Netdata updater
* Only sleep if we're not a tty (e.g: cron) and use a random interval between 30m-60m
* Set lower bound to 1s
* Disable random sleep / netdata-updater splay in lifecycle tests
| -rwxr-xr-x | packaging/installer/netdata-updater.sh | 12 | ||||
| -rwxr-xr-x | tests/updater_checks.bats | 3 |
2 files changed, 13 insertions, 2 deletions
diff --git a/packaging/installer/netdata-updater.sh b/packaging/installer/netdata-updater.sh index 781575c711..38f91aec26 100755 --- a/packaging/installer/netdata-updater.sh +++ b/packaging/installer/netdata-updater.sh @@ -185,8 +185,8 @@ update() { do_not_start="--dont-start-it" fi - if [ -n "${NETDATA_SELECTED_DASHBOARD}" ] ; then - env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}" + if [ -n "${NETDATA_SELECTED_DASHBOARD}" ]; then + env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}" fi info "Re-installing netdata..." @@ -210,6 +210,14 @@ tmpdir= trap cleanup EXIT +# Random sleep to aileviate stampede effect of Agents upgrading +# and disconnecting/reconnecting at the same time (or near to). +# But only we're not a controlling terminal (tty) +# Randomly sleep between 1s and 60m +if [ ! -t 1 ]; then + sleep $(((RANDOM % 3600) + 1))s +fi + # Usually stored in /etc/netdata/.environment : "${ENVIRONMENT_FILE:=THIS_SHOULD_BE_REPLACED_BY_INSTALLER_SCRIPT}" diff --git a/tests/updater_checks.bats b/tests/updater_checks.bats index 6444c081cc..4f39270776 100755 --- a/tests/updater_checks.bats +++ b/tests/updater_checks.bats @@ -56,6 +56,9 @@ setup() { # Run the updater, with the override so that it uses the local repo we have at hand # Try to run the installed, if any, otherwise just run the one from the repo export NETDATA_LOCAL_TARBAL_OVERRIDE="${PWD}" + # Disable random sleep / splay for netdata-updater to avoid sampede effect + # of many agents (dis|re)connecting too quickly all at onace to Netdata Cloud + unset RANDOM; export RANDOM=0 /etc/cron.daily/netdata-updater || ./packaging/installer/netdata-updater.sh ! grep "new_installation" "${ENV}" } |