Retry clone and docker build against transient runner network flakes
CI runners intermittently lose egress (DNS SERVFAILs, refused connects to the git server), which killed builds on transient failures. Retry the checkout clone (3 attempts, 10/20s backoff) and the docker build (BuildKit layer cache makes a retry cheap) before failing the job.
This commit is contained in:
+24
-6
@@ -20,9 +20,27 @@ runs:
|
||||
steps:
|
||||
- shell: bash
|
||||
run: |
|
||||
if [ "${{ inputs.depth }}" = "0" ]; then
|
||||
git clone "https://oauth2:${{ inputs.token }}@git.flowmade.one/${{ inputs.repository }}.git" .
|
||||
else
|
||||
git clone --depth "${{ inputs.depth }}" --branch "${{ inputs.ref }}" \
|
||||
"https://oauth2:${{ inputs.token }}@git.flowmade.one/${{ inputs.repository }}.git" .
|
||||
fi
|
||||
URL="https://oauth2:${{ inputs.token }}@git.flowmade.one/${{ inputs.repository }}.git"
|
||||
clone() {
|
||||
if [ "${{ inputs.depth }}" = "0" ]; then
|
||||
git clone "$URL" .
|
||||
else
|
||||
git clone --depth "${{ inputs.depth }}" --branch "${{ inputs.ref }}" "$URL" .
|
||||
fi
|
||||
}
|
||||
|
||||
# Runner egress is flaky (transient DNS/connection failures to the git
|
||||
# server); retry with backoff before failing the job.
|
||||
ok=0
|
||||
for attempt in 1 2 3; do
|
||||
if clone; then
|
||||
ok=1
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -eq 3 ]; then
|
||||
break
|
||||
fi
|
||||
echo "clone failed, retrying in $((attempt * 10))s..."
|
||||
sleep $((attempt * 10))
|
||||
done
|
||||
[ "$ok" -eq 1 ] || exit 1
|
||||
|
||||
@@ -56,7 +56,18 @@ runs:
|
||||
done <<< "$BUILD_ARGS"
|
||||
fi
|
||||
|
||||
# Build the image
|
||||
docker build $TAG_ARGS $BUILD_ARG_ARGS -f "$DOCKERFILE" "$CONTEXT"
|
||||
# Build the image; retry on failure. Base-image pulls from external
|
||||
# registries flake on transient runner DNS/network issues, and the
|
||||
# BuildKit layer cache makes a retry of a failed build cheap.
|
||||
for attempt in 1 2; do
|
||||
if docker build $TAG_ARGS $BUILD_ARG_ARGS -f "$DOCKERFILE" "$CONTEXT"; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -eq 2 ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo "build failed, retrying in 30s..."
|
||||
sleep 30
|
||||
done
|
||||
|
||||
echo "images=$IMAGES" >> $GITHUB_OUTPUT
|
||||
|
||||
Reference in New Issue
Block a user