Skip to content

Slow down rollout by adding a sleep time after each container creation to allow services a 'settling' time and avoid resource spikes #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
arjoonn-s wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Slow down rollout by adding a sleep time after each container creation to allow services a 'settling' time and avoid resource spikes #66

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 74 additions & 6 deletions docker-rollout
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ VERSION=v0.13
HEALTHCHECK_TIMEOUT=60
NO_HEALTHCHECK_TIMEOUT=10
WAIT_AFTER_HEALTHY_DELAY=0
ROLLING_SLEEP_SECONDS=0

# Print metadata for Docker CLI plugin
if [ "$1" = "docker-cli-plugin-metadata" ]; then
Expand Down Expand Up @@ -58,6 +59,10 @@ Options:
before stopping old container (default: $NO_HEALTHCHECK_TIMEOUT seconds)
--wait-after-healthy N When healthcheck is defined and succeeds, wait for additional N seconds
before stopping the old container (default: 0 seconds)
--rolling-sleep-seconds N Sleep N seconds after starting each new container before starting
the next one. If any new container becomes unhealthy during the sleep,
the rollout aborts and new containers are removed (default: 0, disabled).
Has no effect when scale=1.
--env-file FILE Specify an alternate environment file
-p, --project-name NAME Specify an alternate project name
--profile NAME Specify an alternate profile to use
Expand All @@ -77,6 +82,11 @@ healthcheck() {
docker $DOCKER_ARGS inspect --format='{{json .State.Health.Status}}' "$1" | grep -v "unhealthy" | grep -q "healthy"
}

is_unhealthy() {
# shellcheck disable=SC2086 # DOCKER_ARGS must be unquoted to allow multiple arguments
docker $DOCKER_ARGS inspect --format='{{json .State.Health.Status}}' "$1" | grep -q '"unhealthy"'
}

scale() {
# shellcheck disable=SC2086 # COMPOSE_FILES and ENV_FILES must be unquoted to allow multiple files
$COMPOSE_COMMAND $COMPOSE_FILES $ENV_FILES up --detach --scale "$1=$2" --no-recreate "$1"
Expand All @@ -94,13 +104,67 @@ main() {
OLD_CONTAINER_IDS_STRING=$($COMPOSE_COMMAND $COMPOSE_FILES $ENV_FILES ps --quiet "$SERVICE" | tr '\n' '|' | sed 's/|$//')
OLD_CONTAINER_IDS=$(echo "$OLD_CONTAINER_IDS_STRING" | tr '|' ' ')
SCALE=$(echo "$OLD_CONTAINER_IDS" | wc -w | tr -d ' ')
SCALE_TIMES_TWO=$((SCALE * 2))
echo "==> Scaling '$SERVICE' to '$SCALE_TIMES_TWO' instances"
scale "$SERVICE" $SCALE_TIMES_TWO

# Create a variable that contains the IDs of the new containers, but not the old ones
# shellcheck disable=SC2086 # COMPOSE_FILES and ENV_FILES must be unquoted to allow multiple files
NEW_CONTAINER_IDS=$($COMPOSE_COMMAND $COMPOSE_FILES $ENV_FILES ps --quiet "$SERVICE" | grep -Ev "$OLD_CONTAINER_IDS_STRING" | tr '\n' ' ')
if [ "$ROLLING_SLEEP_SECONDS" = "0" ] || [ "$SCALE" = "1" ]; then
SCALE_TIMES_TWO=$((SCALE * 2))
echo "==> Scaling '$SERVICE' to '$SCALE_TIMES_TWO' instances"
scale "$SERVICE" $SCALE_TIMES_TWO

# Create a variable that contains the IDs of the new containers, but not the old ones
# shellcheck disable=SC2086 # COMPOSE_FILES and ENV_FILES must be unquoted to allow multiple files
NEW_CONTAINER_IDS=$($COMPOSE_COMMAND $COMPOSE_FILES $ENV_FILES ps --quiet "$SERVICE" | grep -Ev "$OLD_CONTAINER_IDS_STRING" | tr '\n' ' ')
else
echo "==> Rolling out '$SERVICE' ($SCALE instances, ${ROLLING_SLEEP_SECONDS}s sleep between containers)"
HAS_HEALTHCHECK="false"

for i in $(seq 1 "$SCALE"); do
echo "==> Creating new container $i/$SCALE"
scale "$SERVICE" $((SCALE + i))

# shellcheck disable=SC2086 # COMPOSE_FILES and ENV_FILES must be unquoted to allow multiple files
NEW_CONTAINER_IDS=$($COMPOSE_COMMAND $COMPOSE_FILES $ENV_FILES ps --quiet "$SERVICE" | grep -Ev "$OLD_CONTAINER_IDS_STRING" | tr '\n' ' ')

# Detect healthcheck once from the first new container
if [ "$i" = "1" ]; then
# shellcheck disable=SC2086 # DOCKER_ARGS must be unquoted to allow multiple arguments
if docker $DOCKER_ARGS inspect --format='{{json .State.Health}}' "$(echo $NEW_CONTAINER_IDS | cut -d\ -f 1)" | grep -q "Status"; then
HAS_HEALTHCHECK="true"
fi
fi

if [ "$i" != "$SCALE" ]; then
echo "==> Sleeping ${ROLLING_SLEEP_SECONDS}s before next container"
sleep "$ROLLING_SLEEP_SECONDS"

if [ "$HAS_HEALTHCHECK" = "true" ]; then
UNHEALTHY_FOUND=0
for NEW_CONTAINER_ID in $NEW_CONTAINER_IDS; do
if is_unhealthy "$NEW_CONTAINER_ID"; then
UNHEALTHY_FOUND=1
break
fi
done

if [ "$UNHEALTHY_FOUND" = "1" ]; then
for NEW_CONTAINER_ID in $NEW_CONTAINER_IDS; do
echo "==> Health check status for container $NEW_CONTAINER_ID"
# shellcheck disable=SC2086 # DOCKER_ARGS must be unquoted to allow multiple arguments
docker $DOCKER_ARGS inspect --format='{{json .State.Health}}' "$NEW_CONTAINER_ID"
echo "==> Logs for container $NEW_CONTAINER_ID"
# shellcheck disable=SC2086 # DOCKER_ARGS must be unquoted to allow multiple arguments
docker $DOCKER_ARGS logs "$NEW_CONTAINER_ID"
done
echo "==> New containers are unhealthy. Rolling back." >&2
# shellcheck disable=SC2086 # DOCKER_ARGS and NEW_CONTAINER_IDS must be unquoted to allow multiple arguments
docker $DOCKER_ARGS stop $NEW_CONTAINER_IDS
# shellcheck disable=SC2086 # DOCKER_ARGS and NEW_CONTAINER_IDS must be unquoted to allow multiple arguments
docker $DOCKER_ARGS rm $NEW_CONTAINER_IDS
exit 1
fi
fi
fi
done
fi

# Check if first container has healthcheck
# shellcheck disable=SC2086 # DOCKER_ARGS must be unquoted to allow multiple arguments
Expand Down Expand Up @@ -222,6 +286,10 @@ while [ $# -gt 0 ]; do
PRE_STOP_HOOK="$2"
shift 2
;;
--rolling-sleep-seconds)
ROLLING_SLEEP_SECONDS="$2"
shift 2
;;
-v | --version)
echo "docker-rollout version $VERSION"
exit 0
Expand Down