Skip to content

Jenkins: retry checkout scm with backoff and debug diagnostics on failure #5186

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
causten wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

Jenkins: retry checkout scm with backoff and debug diagnostics on failure #5186

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1236f59
Jenkins: retry checkout scm with backoff and debug diagnostics on fai…
causten Aug 24, 2026
77971e0
Jenkins: fix interrupt propagation and credential leak in checkoutWit…
causten Aug 25, 2026
f525494
Jenkins: drop git remote -v and filter .url keys from git config diag…
causten Aug 25, 2026
184a7cf
Merge branch 'develop' into jenkins-checkout-retry
causten Aug 28, 2026
bd1c600
Fix checkoutWithRetry: serializable extensions list, abort rethrow, c…
causten Sep 1, 2026
24654f7
CI: use shallow clone in checkoutWithRetry to avoid fetch timeout
causten Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 43 additions & 5 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,44 @@ def resolveSha() {
return null
}

def checkoutWithRetry = { int maxAttempts = 3 ->
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
try {
checkout([
$class: 'GitSCM',
branches: scm.branches,
// new ArrayList() ensures the extensions list is a serializable
// type; concatenating directly onto scm.extensions (a live SCM
// descriptor) can produce a non-serializable result that causes
// the workspace to appear empty after an apparently successful clone.
extensions: new ArrayList(scm.extensions) + [
[$class: 'CloneOption', timeout: 30, shallow: true, depth: 1, retryFetchCount: 3],
[$class: 'CheckoutOption', timeout: 30],
],
userRemoteConfigs: scm.userRemoteConfigs,
])
return
} catch (Exception e) {
if (e instanceof InterruptedException ||
e.class.name == 'org.jenkinsci.plugins.workflow.steps.FlowInterruptedException' ||
e.class.name == 'hudson.AbortException') {
throw e
}
if (attempt == maxAttempts) {
// Exclude url keys to avoid leaking embedded credentials.
sh 'git config --list | grep -v -i "token\\|password\\|auth\\|header\\|url" || true'
sh 'ip route || true'
sh 'curl -sv https://github.com 2>&1 | tail -20 || true'
throw e
}
// Linear backoff: 15 s, 30 s, 45 s, ...
def delay = 15 * attempt
echo "checkout scm failed (attempt ${attempt}/${maxAttempts}): ${e.message}. Retrying in ${delay}s..."
sleep(time: delay, unit: 'SECONDS')
}
}
}

def autoSetGitStatus = { Map conf = [:], Closure body ->
def statusContext = conf.get("gitHubContext", "Unknown")
def description = conf.get("description", "Building")
Expand Down Expand Up @@ -256,7 +294,7 @@ def rocmtest = { Map conf = [:], Closure body ->
def docker_opts
stage("setup ${variant}") {
sh 'printenv'
checkout scm
checkoutWithRetry()
setup()

def video_id = sh(returnStdout: true, script: 'getent group video | cut -d: -f3').trim()
Expand Down Expand Up @@ -317,7 +355,7 @@ pipeline {
withCredentials([usernamePassword(credentialsId: 'docker_test_cred', passwordVariable: 'DOCKERHUB_PASS', usernameVariable: 'DOCKERHUB_USER')]) {
sh "echo $DOCKERHUB_PASS | docker login --username $DOCKERHUB_USER --password-stdin"
sh 'printenv'
checkout scm
checkoutWithRetry()
def calculateImageTagScript = """
shopt -s globstar
sha256sum Dockerfile **/*requirements.txt tools/requirements-py.txt **/install_prereqs.sh **/rbuild.ini **/test/onnx/.onnxrt-commit | sha256sum | cut -d " " -f 1
Expand All @@ -340,7 +378,7 @@ pipeline {
autoSetGitStatus(credentialsId: "${env.migraphx_ci_creds}", gitHubContext: "Jenkins - Build image", account: 'ROCm', repo: 'AMDMIGraphX', description: 'Building image', failureDescription: 'Failed to build image', successDescription: 'Image build succeeded') {
withCredentials([usernamePassword(credentialsId: 'docker_test_cred', passwordVariable: 'DOCKERHUB_PASS', usernameVariable: 'DOCKERHUB_USER')]) {
sh "echo $DOCKERHUB_PASS | docker login --username $DOCKERHUB_USER --password-stdin"
checkout scm
checkoutWithRetry()
def builtImage

try {
Expand Down Expand Up @@ -507,7 +545,7 @@ pipeline {
withCredentials([usernamePassword(credentialsId: 'docker_test_cred', passwordVariable: 'DOCKERHUB_PASS', usernameVariable: 'DOCKERHUB_USER')]) {
sh "echo $DOCKERHUB_PASS | docker login --username $DOCKERHUB_USER --password-stdin"
sh 'printenv'
checkout scm
checkoutWithRetry()
def calculateOrtImageTagScript = """
sha256sum tools/docker/ort.dockerfile test/onnx/.onnxrt-commit tools/build_and_test_onnxrt.sh tools/pai_test_launcher.sh tools/pai_provider_test_launcher.sh | sha256sum | cut -d " " -f 1
"""
Expand All @@ -528,7 +566,7 @@ pipeline {
autoSetGitStatus(credentialsId: "${env.migraphx_ci_creds}", gitHubContext: "Jenkins - Build ORT image", account: 'ROCm', repo: 'AMDMIGraphX', description: 'Building ORT image', failureDescription: 'Failed to build ORT image', successDescription: 'ORT image build succeeded') {
withCredentials([usernamePassword(credentialsId: 'docker_test_cred', passwordVariable: 'DOCKERHUB_PASS', usernameVariable: 'DOCKERHUB_USER')]) {
sh "echo $DOCKERHUB_PASS | docker login --username $DOCKERHUB_USER --password-stdin"
checkout scm
checkoutWithRetry()
def builtOrtImage

try {
Expand Down
Loading