From 44b8491b4c6aa6ce7af0234464bdac17ff15b408 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:56:55 -0400 Subject: [PATCH 1/3] fix(ci): hold a stacked PR until the older PR it builds on lands A PR stacked on an older open PR targets main but holds that PR's commits. When the train held the older PR on a review thread it went on to the next one: landed first, that PR would have carried the older one's work into main under its own title (#337 over #335, #339 over #337 were drafted by hand to stop it), and its refresh merged main over the older PR's commits, so after that PR landed its merge base no longer applied and a lane had to merge it by hand. The train now sends such a PR to the end of the queue while an older open PR (a lower number) shares commits with it that main does not have. On the queue of 2026-10-28: #324 waits for #323, #355 for #324, #356 for #355, #410 for #356, #412 for #410, #339 for #337 and #341 for #339; #323, #337, #389 and #296 wait for nothing. --- tools/maintainer/train/train.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/maintainer/train/train.sh b/tools/maintainer/train/train.sh index 45561b5b..e3896f1a 100755 --- a/tools/maintainer/train/train.sh +++ b/tools/maintainer/train/train.sh @@ -7,7 +7,8 @@ # refresh.sh merges main into each PR and pushes, land.sh waits for its GitHub Actions run, commits # the corpus counts that run measured when they moved, and lands it. Nothing is built or tested on # this machine. A PR that went DIRTY or found main moved under it is refreshed once more. A PR -# stacked on another PR's branch waits at the end of the queue until GitHub retargets it to main. +# stacked on another PR's branch waits at the end of the queue until GitHub retargets it to main, +# and so does one on main that holds commits of an older open PR, until that PR lands. # Anything else that stops a PR is logged and the train moves on. Append to the queue file to add # work while it runs; this script is read once, so restart the train to pick up a change to it. # Progress: $TRAIN_STATE/train.log. @@ -25,6 +26,22 @@ wait_at_end() { log "TRAIN #$P $1" [ "$waiting" -ge "$(grep -c -E '^[0-9]+$' "$Q")" ] && { log "TRAIN every queued PR waits; waiting ${TRAIN_DEFER_SLEEP:-600} s"; sleep "${TRAIN_DEFER_SLEEP:-600}"; waiting=0; } } +# A PR stacked on an older open PR targets main but holds that PR's commits. Landed first, it would +# carry the older PR's work into main under its own title; refreshed while the older PR is held, it +# merges main over the older PR's commits and can no longer merge against them once that PR lands. +# It waits until the older PR (a lower number) holding commits main does not have lands or closes. +older_open() { + local head n h c + git -C "$TRAIN_CLONE" fetch -q origin main 2> /dev/null && git -C "$TRAIN_CLONE" fetch -q origin "pull/$1/head" 2> /dev/null || return + head=$(git -C "$TRAIN_CLONE" rev-parse FETCH_HEAD) + gh pr list -R "$TRAIN_REPO" --state open --base main --limit 200 --json number,headRefOid --jq ".[]|select(.number < $1)|\"\(.number) \(.headRefOid)\"" 2> /dev/null | + while read -r n h; do + git -C "$TRAIN_CLONE" cat-file -e "$h^{commit}" 2> /dev/null || git -C "$TRAIN_CLONE" fetch -q origin "pull/$n/head" 2> /dev/null || continue + for c in $(git -C "$TRAIN_CLONE" merge-base --all "$h" "$head" 2> /dev/null); do + git -C "$TRAIN_CLONE" merge-base --is-ancestor "$c" origin/main || { echo "$n"; exit 0; } + done + done +} while :; do P=$(grep -m1 -E '^[0-9]+$' "$Q") [ -z "$P" ] && break @@ -33,6 +50,8 @@ while :; do read -r st base < <(gh pr view "$P" -R "$TRAIN_REPO" --json state,baseRefName --jq '"\(.state) \(.baseRefName)"' 2>/dev/null) [ "$st" != OPEN ] && { log "TRAIN #$P skipped: state=$st"; break; } [ "$base" != main ] && { wait_at_end "waits: it targets $base, not main; back to the end of the queue"; break; } + under=$(older_open "$P") + [ -n "$under" ] && { wait_at_end "waits: it holds commits of open #$under, which lands first; back to the end of the queue"; break; } log "TRAIN #$P refresh (attempt $attempt)" out=$(bash "$TRAIN_HERE/refresh.sh" "$P" 2>&1); rc=$? echo "$out" > "$TRAIN_STATE/refresh-$P.log" From 32b43e977a84d8955d3452f0713cce8389e7af07 Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:06:50 -0400 Subject: [PATCH 2/3] fix(ci): check every open PR, skip landed work, and defer on a failed lookup The stack check listed at most 200 open PRs, only those targeting main, treated any shared commit as outstanding even when it belonged to a PR that had already landed by squash, and let a PR through when a fetch or gh call failed. It now fetches every PR head to refs/train/pr/ in the main clone, pages through all open PRs whatever they target, ignores a shared commit that a merged PR's head holds, and returns 2 on any failed lookup, which sends the PR to the end of the queue. Checked in a scratch repository (two PRs built on a landed PR do not wait for each other; a PR built on another waits for it; a failing gh returns 2) and on the live queue, where the R13 and unp stacks give the same order as before. --- tools/maintainer/train/train.sh | 34 +++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tools/maintainer/train/train.sh b/tools/maintainer/train/train.sh index e3896f1a..d0e8b72d 100755 --- a/tools/maintainer/train/train.sh +++ b/tools/maintainer/train/train.sh @@ -29,16 +29,29 @@ wait_at_end() { # A PR stacked on an older open PR targets main but holds that PR's commits. Landed first, it would # carry the older PR's work into main under its own title; refreshed while the older PR is held, it # merges main over the older PR's commits and can no longer merge against them once that PR lands. -# It waits until the older PR (a lower number) holding commits main does not have lands or closes. +# It waits while an older open PR (a lower number, whatever branch it targets) shares a commit with it +# that is neither on main nor in a PR that has already landed by squash: two PRs built on the same +# landed PR share only that PR's commits and do not wait for each other. Every PR head is fetched to +# refs/train/pr/ in the main clone. Prints the PR to wait for, or nothing; any failed lookup +# returns 2, and the caller then defers the PR rather than letting it through unchecked. +g() { git -C "$TRAIN_CLONE" "$@"; } older_open() { - local head n h c - git -C "$TRAIN_CLONE" fetch -q origin main 2> /dev/null && git -C "$TRAIN_CLONE" fetch -q origin "pull/$1/head" 2> /dev/null || return - head=$(git -C "$TRAIN_CLONE" rev-parse FETCH_HEAD) - gh pr list -R "$TRAIN_REPO" --state open --base main --limit 200 --json number,headRefOid --jq ".[]|select(.number < $1)|\"\(.number) \(.headRefOid)\"" 2> /dev/null | - while read -r n h; do - git -C "$TRAIN_CLONE" cat-file -e "$h^{commit}" 2> /dev/null || git -C "$TRAIN_CLONE" fetch -q origin "pull/$n/head" 2> /dev/null || continue - for c in $(git -C "$TRAIN_CLONE" merge-base --all "$h" "$head" 2> /dev/null); do - git -C "$TRAIN_CLONE" merge-base --is-ancestor "$c" origin/main || { echo "$n"; exit 0; } + local head open merged="" n h c pr + g fetch -q origin main '+refs/pull/*/head:refs/train/pr/*' 2> /dev/null || return 2 + head=$(g rev-parse -q --verify "refs/train/pr/$1^{commit}") || return 2 + open=$(gh api --paginate "repos/$TRAIN_REPO/pulls?state=open&per_page=100" --jq ".[]|select(.number < $1)|.number" 2> /dev/null) || return 2 + for n in $open; do + h=$(g rev-parse -q --verify "refs/train/pr/$n^{commit}") || return 2 + for c in $(g merge-base --all "$h" "$head"); do + g merge-base --is-ancestor "$c" origin/main && continue + if [ -z "$merged" ]; then + merged=$(gh api --paginate "repos/$TRAIN_REPO/pulls?state=closed&per_page=100" --jq '.[]|select(.merged_at)|.number' 2> /dev/null) || return 2 + fi + for pr in $(g for-each-ref --contains "$c" --format='%(refname:lstrip=3)' refs/train/pr/); do + grep -qx "$pr" <<< "$merged" && continue 2 + done + echo "$n" + return 0 done done } @@ -50,7 +63,8 @@ while :; do read -r st base < <(gh pr view "$P" -R "$TRAIN_REPO" --json state,baseRefName --jq '"\(.state) \(.baseRefName)"' 2>/dev/null) [ "$st" != OPEN ] && { log "TRAIN #$P skipped: state=$st"; break; } [ "$base" != main ] && { wait_at_end "waits: it targets $base, not main; back to the end of the queue"; break; } - under=$(older_open "$P") + under=$(older_open "$P"); orc=$? + [ $orc != 0 ] && { wait_at_end "waits: could not check which open PRs it builds on; back to the end of the queue"; break; } [ -n "$under" ] && { wait_at_end "waits: it holds commits of open #$under, which lands first; back to the end of the queue"; break; } log "TRAIN #$P refresh (attempt $attempt)" out=$(bash "$TRAIN_HERE/refresh.sh" "$P" 2>&1); rc=$? From f1038c362f02bea0ffc507a0630d0e49c792048a Mon Sep 17 00:00:00 2001 From: Griffin Long <151978260+DrunkOnJava@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:39:22 -0400 Subject: [PATCH 3/3] fix(ci): count only PRs merged into main as landed when checking a stack A PR merged into a feature branch has merged_at set too, so its commits were treated as landed and a newer PR holding them went ahead of the open PR they were waiting in. The exemption now takes only PRs whose base is main. All 187 merged PRs so far target main, so the live order is unchanged. --- tools/maintainer/train/train.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/maintainer/train/train.sh b/tools/maintainer/train/train.sh index d0e8b72d..b052d545 100755 --- a/tools/maintainer/train/train.sh +++ b/tools/maintainer/train/train.sh @@ -30,10 +30,11 @@ wait_at_end() { # carry the older PR's work into main under its own title; refreshed while the older PR is held, it # merges main over the older PR's commits and can no longer merge against them once that PR lands. # It waits while an older open PR (a lower number, whatever branch it targets) shares a commit with it -# that is neither on main nor in a PR that has already landed by squash: two PRs built on the same -# landed PR share only that PR's commits and do not wait for each other. Every PR head is fetched to -# refs/train/pr/ in the main clone. Prints the PR to wait for, or nothing; any failed lookup -# returns 2, and the caller then defers the PR rather than letting it through unchecked. +# that is neither on main nor in a PR that has already landed on main by squash (one merged into +# another branch has not): two PRs built on the same landed PR share only that PR's commits and do +# not wait for each other. Every PR head is fetched to refs/train/pr/ in the main clone. Prints +# the PR to wait for, or nothing; any failed lookup returns 2, and the caller then defers the PR +# rather than letting it through unchecked. g() { git -C "$TRAIN_CLONE" "$@"; } older_open() { local head open merged="" n h c pr @@ -45,7 +46,7 @@ older_open() { for c in $(g merge-base --all "$h" "$head"); do g merge-base --is-ancestor "$c" origin/main && continue if [ -z "$merged" ]; then - merged=$(gh api --paginate "repos/$TRAIN_REPO/pulls?state=closed&per_page=100" --jq '.[]|select(.merged_at)|.number' 2> /dev/null) || return 2 + merged=$(gh api --paginate "repos/$TRAIN_REPO/pulls?state=closed&per_page=100" --jq '.[]|select(.merged_at and .base.ref == "main")|.number' 2> /dev/null) || return 2 fi for pr in $(g for-each-ref --contains "$c" --format='%(refname:lstrip=3)' refs/train/pr/); do grep -qx "$pr" <<< "$merged" && continue 2