From ecf7a58697502fb009092eacdbade19164a50bf4 Mon Sep 17 00:00:00 2001
From: JRPAN <25518778+JRPan@users.noreply.github.com>
Date: Sat, 31 Oct 2026 01:30:50 -0400
Subject: [PATCH 1/5] Move the CI Dockerfiles into dockerfiles/ and build them
 from this repo

Every image branch of accel-sim/Dockerfile becomes dockerfiles/<tag>.Dockerfile,
with 13.2-minimal taken from accel-sim/Dockerfile#12 (the branch of that name
held a 12.8 Dockerfile). The two never-published 2025 images (18.04/11.0,
22.04/11.7) are left behind. 12.8-minimal gains the AWS CLI and boto3.

docker-images.yml builds only the Dockerfiles a change touches: a modified file
always, an added one only if its tag is not on ghcr yet. PRs build without
pushing; the merge to dev rebuilds from pc02's cache and pushes. Builds run on
tgrogers-raid with DOCKER_HOST pointing at tgrogers-pc02's daemon.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/scripts/docker-plan.sh                | 46 ++++++++++
 .github/workflows/docker-images.yml           | 83 +++++++++++++++++++
 dockerfiles/README.md                         | 29 +++++++
 ....8-riscv-gnu-2024.08.06-nightly.Dockerfile | 67 +++++++++++++++
 ...buntu-22.04-cuda-11.7.1-minimal.Dockerfile | 35 ++++++++
 .../Ubuntu-24.04-cuda-12.8-minimal.Dockerfile | 46 ++++++++++
 .../Ubuntu-24.04-cuda-12.8-vllm.Dockerfile    | 56 +++++++++++++
 dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile | 47 +++++++++++
 .../Ubuntu-24.04-cuda-12.9-minimal.Dockerfile | 33 ++++++++
 dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile | 47 +++++++++++
 ...buntu-24.04-cuda-13.1.1-minimal.Dockerfile | 35 ++++++++
 .../Ubuntu-24.04-cuda-13.2-minimal.Dockerfile | 44 ++++++++++
 12 files changed, 568 insertions(+)
 create mode 100755 .github/scripts/docker-plan.sh
 create mode 100644 .github/workflows/docker-images.yml
 create mode 100644 dockerfiles/README.md
 create mode 100644 dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-12.8-vllm.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile

diff --git a/.github/scripts/docker-plan.sh b/.github/scripts/docker-plan.sh
new file mode 100755
index 0000000000..16bde80d86
--- /dev/null
+++ b/.github/scripts/docker-plan.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Decide which dockerfiles/<tag>.Dockerfile images to build. Prints a JSON array of tags on stdout,
+# and one "build/skip <tag>: <reason>" line per image on stderr (and in the job summary).
+#   workflow_dispatch: the tags in $INPUT_IMAGES (space separated).
+#   pull_request / push: Dockerfiles changed between $BASE and $HEAD. A modified file always
+#   builds; an added one builds only if its tag is not on ghcr yet, so importing or renaming
+#   existing images does not rebuild them.
+# Env: EVENT, BASE, HEAD, INPUT_IMAGES, IMAGE (ghcr repository, e.g. accel-sim/accel-sim-framework)
+set -euo pipefail
+
+on_ghcr() {
+  local token code
+  token=$(curl -fsS "https://ghcr.io/token?scope=repository:$IMAGE:pull&service=ghcr.io" |
+    python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')
+  code=$(curl -s -o /dev/null -w '%{http_code}' -I -H "Authorization: Bearer $token" \
+    -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json' \
+    "https://ghcr.io/v2/$IMAGE/manifests/$1")
+  case $code in
+    200) return 0 ;;
+    404) return 1 ;;
+    *) echo "ghcr manifest check for $1 returned HTTP $code (expected 200 or 404)" >&2; exit 1 ;;
+  esac
+}
+
+tags=()
+say() { echo "$*" >&2; [ -z "${GITHUB_STEP_SUMMARY:-}" ] || echo "- $*" >> "$GITHUB_STEP_SUMMARY"; }
+if [ "$EVENT" = workflow_dispatch ]; then
+  for t in $INPUT_IMAGES; do
+    [ -f "dockerfiles/$t.Dockerfile" ] || { echo "no dockerfiles/$t.Dockerfile" >&2; exit 1; }
+    tags+=("$t"); say "build $t: requested"
+  done
+else
+  [[ $BASE =~ ^0+$ ]] && BASE=$HEAD~1  # first push of a branch has no "before"
+  while read -r status path; do
+    t=$(basename "$path" .Dockerfile)
+    case $status in
+      M) tags+=("$t"); say "build $t: modified" ;;
+      A) if on_ghcr "$t"; then say "skip $t: added, already on ghcr"; else tags+=("$t"); say "build $t: new image"; fi ;;
+      D) say "skip $t: deleted" ;;
+      *) echo "unexpected git status $status for $path" >&2; exit 1 ;;
+    esac
+  done < <(git diff --no-renames --name-status "$BASE" "$HEAD" -- 'dockerfiles/*.Dockerfile')
+fi
+[ ${#tags[@]} -gt 0 ] || say "nothing to build"
+
+printf '%s\n' "${tags[@]}" | python3 -c 'import json,sys; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))'
diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
new file mode 100644
index 0000000000..66db2f5091
--- /dev/null
+++ b/.github/workflows/docker-images.yml
@@ -0,0 +1,83 @@
+# Builds the CI images in dockerfiles/ (one <tag>.Dockerfile per ghcr tag).
+# PR: build the changed images only, no push. Merge to dev: build again (a cache hit on
+# tgrogers-pc02, so only the push costs time) and push. Manual: rebuild the named images.
+# Which images count as changed: .github/scripts/docker-plan.sh.
+name: Docker images
+
+on:
+  pull_request:
+    paths: ["dockerfiles/*.Dockerfile"]
+  push:
+    branches: [dev]
+    paths: ["dockerfiles/*.Dockerfile"]
+  workflow_dispatch:
+    inputs:
+      images:
+        description: "Tags to build (dockerfiles/<tag>.Dockerfile), space separated"
+        required: true
+        type: string
+      no_cache:
+        description: "Rebuild every layer (e.g. to pick up a newer gpu-app-collection)"
+        type: boolean
+        default: false
+
+env:
+  IMAGE: accel-sim/accel-sim-framework
+
+jobs:
+  plan:
+    if: github.repository == 'accel-sim/accel-sim-framework'
+    runs-on: tgrogers-raid
+    outputs:
+      images: ${{ steps.plan.outputs.images }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Pick images to build
+        id: plan
+        env:
+          EVENT: ${{ github.event_name }}
+          # A PR checks out its merge commit, whose first parent is the base branch.
+          BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}
+          HEAD: ${{ github.event_name == 'pull_request' && 'HEAD' || github.sha }}
+          INPUT_IMAGES: ${{ inputs.images }}
+        run: echo "images=$(.github/scripts/docker-plan.sh)" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: plan
+    if: needs.plan.outputs.images != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJSON(needs.plan.outputs.images) }}
+    name: ${{ matrix.image }}
+    # tgrogers-raid has no docker daemon: tgrghci's docker CLI + buildx drive tgrogers-pc02's over ssh.
+    runs-on: tgrogers-raid
+    permissions:
+      contents: read
+      packages: write
+    env:
+      DOCKER_HOST: ssh://tgrogers-pc02
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to ghcr.io
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build${{ github.event_name != 'pull_request' && ' and push' || '' }}
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: dockerfiles
+          file: dockerfiles/${{ matrix.image }}.Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          no-cache: ${{ inputs.no_cache == true }}
+          tags: ghcr.io/${{ env.IMAGE }}:${{ matrix.image }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
diff --git a/dockerfiles/README.md b/dockerfiles/README.md
new file mode 100644
index 0000000000..697d300364
--- /dev/null
+++ b/dockerfiles/README.md
@@ -0,0 +1,29 @@
+# CI container images
+
+Each `<tag>.Dockerfile` here builds one image, `ghcr.io/accel-sim/accel-sim-framework:<tag>`.
+These files used to live one per branch in `accel-sim/Dockerfile`, which is now retired.
+
+The `*-minimal` images are what CI runs (`IMAGE_TAG` in `.github/workflows/main.yml`). They hold:
+- the build dependencies
+- the gpu-app-collection CI apps and their regression data
+- the AWS CLI and boto3, so the image can also run AWS Batch jobs (`util/aws_batch`) — 12.8-minimal only so far
+
+## When images build
+
+`.github/workflows/docker-images.yml` builds only the Dockerfiles a change touches, and lists each one as build or skip with a reason. The rules are in `.github/scripts/docker-plan.sh`:
+- **Pull request:** builds the images whose Dockerfile changed, without pushing.
+  - A modified file always builds.
+  - A new file builds only if its tag is not on ghcr yet.
+- **Merge to `dev`:** the same rules, followed by a push to ghcr.
+  - The build runs on the same docker daemon as the PR's build, so it is a cache hit and the push is the only real work.
+- **Manual:** Actions → Docker images → Run workflow, with the tags to build. Tick `no_cache` to rebuild from scratch, for example to pick up a newer gpu-app-collection.
+
+## Build host
+
+Builds run on a `tgrogers-raid` runner, which has no docker daemon. Its account (`tgrghci`) has the docker CLI and buildx installed. The workflow sets `DOCKER_HOST=ssh://tgrogers-pc02`, so the build runs on tgrogers-pc02's daemon.
+
+To build by hand on any docker host:
+
+```bash
+docker build -f dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile dockerfiles
+```
diff --git a/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile b/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile
new file mode 100644
index 0000000000..0634f98b25
--- /dev/null
+++ b/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile
@@ -0,0 +1,67 @@
+FROM ubuntu:22.04
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+ADD . /accel-sim
+
+ENV CUDA_INSTALL_PATH /usr/local/cuda-11.7
+ENV PTXAS_CUDA_INSTALL_PATH /usr/local/cuda-11.7
+ENV GPUAPPS_ROOT /accel-sim/gpu-app-collection
+ENV LLVM_INSTALL_PATH /accel-sim/llvm-install
+ENV RISCV_TOOLCHAIN_INSTALL_PATH /accel-sim/riscv-gnu-install
+
+# Install CUDA
+RUN apt-get update \
+&& apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python-setuptools python3-pip cmake \
+&& apt-get clean \
+&& pip3 install pyyaml plotly psutil \
+&& wget https://developer.download.nvidia.com/compute/cuda/11.7.0/local_installers/cuda_11.7.0_515.43.04_linux.run \
+&& sh cuda_11.7.0_515.43.04_linux.run --silent --toolkit \
+&& rm cuda_11.7.0_515.43.04_linux.run \
+&& rm -rf /usr/local/cuda-11.7/nsight-compute-2022.2.0 \
+&& rm -rf /usr/local/cuda-11.7/nsight-systems-2022.1.3
+
+# Build LLVM 18.1.8
+RUN git clone https://github.com/llvm/llvm-project.git \
+&& mkdir llvm-install \
+&& cd llvm-project \
+&& git checkout llvmorg-18.1.8 \
+&& mkdir build && cd build \
+&& cmake -DLLVM_TARGETS_TO_BUILD="RISCV;X86;NVPTX" -DLLVM_DEFAULT_TARGET_TRIPLE=riscv64-unknown-linux-gnu \
+         -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;lld" -DCMAKE_INSTALL_PREFIX=$LLVM_INSTALL_PATH ../llvm \
+&& cmake --build . -j4 \
+&& cmake --build . --target install \
+&& cd .. && cd .. \
+&& rm -rf llvm-project
+
+# Build RISCV GNU Toolchain 2024.08.06.nightly
+RUN apt-get update && apt-get -y install autoconf automake autotools-dev curl python3 python3-pip libmpc-dev libmpfr-dev libgmp-dev gawk build-essential bison flex texinfo gperf libtool patchutils bc zlib1g-dev libexpat-dev ninja-build git cmake libglib2.0-dev libslirp-dev
+RUN git clone https://github.com/riscv-collab/riscv-gnu-toolchain.git \
+&& mkdir riscv-gnu-install \
+&& cd riscv-gnu-toolchain \
+&& git checkout 2024.08.06 \
+&& ./configure --prefix=$RISCV_TOOLCHAIN_INSTALL_PATH \
+&& make linux -j4 \
+&& cd .. \
+&& rm -rf riscv-gnu-toolchain
+
+# Setup GPU app collection in SST mode
+# Also remove prebuilt x86 binaries
+# Also remove all data except for rodinia-2.0
+RUN export PATH=$CUDA_INSTALL_PATH/bin:$PATH \
+&& git clone https://github.com/accel-sim/gpu-app-collection \
+&& cd gpu-app-collection \
+&& git pull \
+&& git checkout sst_support \
+&& source ./src/setup_environment sst \
+&& make -j -C ./src data \
+&& mv ./data_dirs/cuda/rodinia/2.0-ft . \
+&& rm -rf ./data_dirs \
+&& mkdir -p ./data_dirs/cuda/rodinia/ \
+&& mv ./2.0-ft ./data_dirs/cuda/rodinia/ \
+&& ls ./data_dirs/cuda/rodinia/ \
+&& rm gpucomputingsdk_4.2.9_linux.run \
+&& rm -rf 4.2
diff --git a/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile b/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile
new file mode 100644
index 0000000000..775ea740c2
--- /dev/null
+++ b/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile
@@ -0,0 +1,35 @@
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 22.04 with CUDA 11.7.1 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+ENV LD_PRELOAD=/lib/x86_64-linux-gnu/libz.so.1
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils && apt-get clean 
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+RUN cd gpu-app-collection && bash test-build.sh ci && bash get_regression_data.sh
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
new file mode 100644
index 0000000000..30d481be28
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
@@ -0,0 +1,46 @@
+FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu24.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 12.8.0 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils \
+      libzstd-dev && apt-get clean
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+# Invoke make directly instead of test-build.sh, which hardcodes a serial
+# `make -C src/ ci`. src/setup_environment is still required: it exports
+# BINDIR/BINSUBDIR/SETENV and the per-CUDA-version gencode flags the
+# suite makefiles read. -j32 propagates to every recursive $(MAKE),
+# including cutlass_mini's cmake-generated makefiles.
+RUN cd gpu-app-collection && source ./src/setup_environment && make -j32 -C src/ ci && bash get_regression_data.sh
+
+# AWS CLI v2 and boto3, so this image can run as an AWS Batch job (accel-sim-framework util/aws_batch)
+RUN apt-get update && apt-get install -y --no-install-recommends unzip && apt-get clean && \
+    wget -qO /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.39.2.zip && \
+    unzip -q /tmp/awscliv2.zip -d /tmp && /tmp/aws/install && rm -rf /tmp/aws /tmp/awscliv2.zip && \
+    aws --version
+RUN pip3 install boto3==1.43.122
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.8-vllm.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.8-vllm.Dockerfile
new file mode 100644
index 0000000000..26db1a7241
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.8-vllm.Dockerfile
@@ -0,0 +1,56 @@
+FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu24.04
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+# NOTE: `bc` and `libzstd-dev` added to the upstream list -- both are load-bearing
+# for the tracer build, which upstream does at runtime from the mounted source:
+#   bc          -> tracer_tool/Makefile shells out to it for the nvcc>=11.7 ARCH check
+#   libzstd-dev -> tracer_tool/traces-processing links -lzstd
+RUN apt-get update && apt-get install -y \
+    wget build-essential xutils-dev bison zlib1g-dev flex \
+    libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev \
+    vim python3-setuptools python3-pip python3-venv cmake \
+    libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+    python3-doc python3-tk python3.12-venv python3.12-doc \
+    binfmt-support psmisc apt-utils gdb curl bash-completion \
+    bc libzstd-dev && \
+    apt-get clean
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+#get Nsys
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update --allow-insecure-repositories && apt update && apt install -y --no-install-recommends gnupg wget \
+    && mkdir -p /etc/apt/keyrings
+RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64/7fa2af80.pub | tee /etc/apt/keyrings/nvidia.asc
+RUN echo "deb [signed-by=/etc/apt/keyrings/nvidia.asc] http://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64 /" | tee /etc/apt/sources.list.d/nvidia.list
+RUN apt-get update --allow-insecure-repositories
+RUN apt install  -y nsight-systems-cli --allow-unauthenticated
+
+# ---------------------------------------------------------------------------
+# vLLM + LLM tracing stack.
+#
+# Everything above is the upstream accel-sim Dockerfile unchanged (bar the two
+# packages noted). Nothing below changes the upstream workflow: no source is
+# baked in, WORKDIR stays /accel-sim, and there is no ENTRYPOINT override -- the
+# nvidia/cuda base entrypoint already does `exec "$@"`, so you pass a command in
+# and the mounted script builds the tracer, exactly as .github/scripts do.
+#
+# This is the single biggest layer in the image (~10 GB).
+# ---------------------------------------------------------------------------
+RUN pip3 install --no-cache-dir vllm transformers nvtx
+RUN pip3 install --no-cache-dir monai diffusers accelerate
\ No newline at end of file
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile
new file mode 100644
index 0000000000..cffcdfb82a
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile
@@ -0,0 +1,47 @@
+FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu24.04
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+RUN apt-get update && apt-get install -y \
+    wget build-essential xutils-dev bison zlib1g-dev flex \
+    libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev \
+    vim python3-setuptools python3-pip python3-venv cmake \
+    libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+    python3-doc python3-tk python3.12-venv python3.12-doc \
+    binfmt-support psmisc apt-utils gdb curl bash-completion && \
+    apt-get clean
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git && cd gpu-app-collection && bash test-build.sh && bash get_regression_data.sh
+
+
+#  autocomplete
+RUN echo "source /usr/share/bash-completion/completions/git" >> ~/.bashrc
+
+# Install fzf
+RUN git clone --depth 1 https://github.com/junegunn/fzf.git ~/.fzf && \
+    ~/.fzf/install --all
+
+#get Nsys
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update --allow-insecure-repositories && apt update && apt install -y --no-install-recommends gnupg wget \
+    && mkdir -p /etc/apt/keyrings 
+RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64/7fa2af80.pub | tee /etc/apt/keyrings/nvidia.asc
+RUN echo "deb [signed-by=/etc/apt/keyrings/nvidia.asc] http://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64 /" | tee /etc/apt/sources.list.d/nvidia.list
+RUN apt-get update --allow-insecure-repositories
+RUN apt install  -y nsight-systems-cli --allow-unauthenticated
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile
new file mode 100644
index 0000000000..34e69d08c5
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile
@@ -0,0 +1,33 @@
+FROM nvidia/cuda:12.9.1-cudnn-devel-ubuntu24.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 12.8.0 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils && apt-get clean 
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+RUN cd gpu-app-collection && bash test-build.sh ci && bash get_regression_data.sh
\ No newline at end of file
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile
new file mode 100644
index 0000000000..226119fa63
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile
@@ -0,0 +1,47 @@
+FROM nvidia/cuda:12.9.1-cudnn-devel-ubuntu24.04
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+RUN apt-get update && apt-get install -y \
+    wget build-essential xutils-dev bison zlib1g-dev flex \
+    libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev \
+    vim python3-setuptools python3-pip python3-venv cmake \
+    libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+    python3-doc python3-tk python3.12-venv python3.12-doc \
+    binfmt-support psmisc apt-utils gdb curl bash-completion && \
+    apt-get clean
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git && cd gpu-app-collection && bash test-build.sh && bash get_regression_data.sh
+
+
+#  autocomplete
+RUN echo "source /usr/share/bash-completion/completions/git" >> ~/.bashrc
+
+# Install fzf
+RUN git clone --depth 1 https://github.com/junegunn/fzf.git ~/.fzf && \
+    ~/.fzf/install --all
+
+#get Nsys
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update --allow-insecure-repositories && apt update && apt install -y --no-install-recommends gnupg wget \
+    && mkdir -p /etc/apt/keyrings 
+RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64/7fa2af80.pub | tee /etc/apt/keyrings/nvidia.asc
+RUN echo "deb [signed-by=/etc/apt/keyrings/nvidia.asc] http://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64 /" | tee /etc/apt/sources.list.d/nvidia.list
+RUN apt-get update --allow-insecure-repositories
+RUN apt install  -y nsight-systems-cli --allow-unauthenticated
diff --git a/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
new file mode 100644
index 0000000000..1ad52f030c
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
@@ -0,0 +1,35 @@
+FROM nvidia/cuda:13.1.1-cudnn-devel-ubuntu24.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 13.1.1 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+ARG GPU_APP_COLLECTION_BRANCH=dev
+
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils && apt-get clean 
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository. 
+RUN git clone --recurse-submodules --branch ${GPU_APP_COLLECTION_BRANCH} --single-branch \
+      https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+RUN cd gpu-app-collection && bash test-build.sh ci && bash get_regression_data.sh
diff --git a/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
new file mode 100644
index 0000000000..48fdad07fd
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
@@ -0,0 +1,44 @@
+FROM nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 13.2.1 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+# Lets the image be validated against a feature branch before that branch
+# lands on dev:  docker build --build-arg GPU_APP_COLLECTION_BRANCH=<branch> .
+ARG GPU_APP_COLLECTION_BRANCH=dev
+
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils \
+      libzstd-dev && apt-get clean
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository
+RUN git clone --recurse-submodules --branch ${GPU_APP_COLLECTION_BRANCH} --single-branch \
+      https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+# Invoke make directly instead of test-build.sh, which hardcodes a serial
+# `make -C src/ ci`. src/setup_environment is still required: it exports
+# BINDIR/BINSUBDIR/SETENV and the per-CUDA-version gencode flags the
+# suite makefiles read. -j32 propagates to every recursive $(MAKE),
+# including cutlass_mini's cmake-generated makefiles.
+RUN cd gpu-app-collection && source ./src/setup_environment && make -j32 -C src/ ci && bash get_regression_data.sh

From 9a708aaa71590f9c3ecaf76a76756b13bcc570b7 Mon Sep 17 00:00:00 2001
From: JRPAN <25518778+JRPan@users.noreply.github.com>
Date: Sat, 31 Oct 2026 01:37:34 -0400
Subject: [PATCH 2/5] Build at most 4 images at once with make -j8 each, to fit
 pc02

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/docker-images.yml                   | 2 ++
 dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile | 4 ++--
 dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile | 4 ++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
index 66db2f5091..ee32c5887a 100644
--- a/.github/workflows/docker-images.yml
+++ b/.github/workflows/docker-images.yml
@@ -49,6 +49,8 @@ jobs:
     if: needs.plan.outputs.images != '[]'
     strategy:
       fail-fast: false
+      # Builds share pc02 (40 cores, 125 GB); the Dockerfiles use make -j8, so 4 at once is 32 jobs.
+      max-parallel: 4
       matrix:
         image: ${{ fromJSON(needs.plan.outputs.images) }}
     name: ${{ matrix.image }}
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
index 30d481be28..bdf0521b97 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
@@ -34,9 +34,9 @@ RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collecti
 # Invoke make directly instead of test-build.sh, which hardcodes a serial
 # `make -C src/ ci`. src/setup_environment is still required: it exports
 # BINDIR/BINSUBDIR/SETENV and the per-CUDA-version gencode flags the
-# suite makefiles read. -j32 propagates to every recursive $(MAKE),
+# suite makefiles read. -j8 propagates to every recursive $(MAKE),
 # including cutlass_mini's cmake-generated makefiles.
-RUN cd gpu-app-collection && source ./src/setup_environment && make -j32 -C src/ ci && bash get_regression_data.sh
+RUN cd gpu-app-collection && source ./src/setup_environment && make -j8 -C src/ ci && bash get_regression_data.sh
 
 # AWS CLI v2 and boto3, so this image can run as an AWS Batch job (accel-sim-framework util/aws_batch)
 RUN apt-get update && apt-get install -y --no-install-recommends unzip && apt-get clean && \
diff --git a/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
index 48fdad07fd..807ffe41ba 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
@@ -39,6 +39,6 @@ RUN git clone --recurse-submodules --branch ${GPU_APP_COLLECTION_BRANCH} --singl
 # Invoke make directly instead of test-build.sh, which hardcodes a serial
 # `make -C src/ ci`. src/setup_environment is still required: it exports
 # BINDIR/BINSUBDIR/SETENV and the per-CUDA-version gencode flags the
-# suite makefiles read. -j32 propagates to every recursive $(MAKE),
+# suite makefiles read. -j8 propagates to every recursive $(MAKE),
 # including cutlass_mini's cmake-generated makefiles.
-RUN cd gpu-app-collection && source ./src/setup_environment && make -j32 -C src/ ci && bash get_regression_data.sh
+RUN cd gpu-app-collection && source ./src/setup_environment && make -j8 -C src/ ci && bash get_regression_data.sh

From 62874d5d0968be6ec420bb757718d695981e3fc9 Mon Sep 17 00:00:00 2001
From: JRPAN <25518778+JRPan@users.noreply.github.com>
Date: Sat, 31 Oct 2026 01:54:24 -0400
Subject: [PATCH 3/5] Drop the 13.1.1-minimal image: CUDA 13 cannot build
 gpu-app-collection's sm_70 targets

It was never published and 13.2-minimal supersedes it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 ...buntu-24.04-cuda-13.1.1-minimal.Dockerfile | 35 -------------------
 1 file changed, 35 deletions(-)
 delete mode 100644 dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile

diff --git a/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
deleted file mode 100644
index 1ad52f030c..0000000000
--- a/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
+++ /dev/null
@@ -1,35 +0,0 @@
-FROM nvidia/cuda:13.1.1-cudnn-devel-ubuntu24.04
-LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
-LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 13.1.1 and CUDNN, for CI runs only"
-
-SHELL ["/bin/bash", "-c"]
-
-WORKDIR /accel-sim
-
-ENV CUDA_INSTALL_PATH=/usr/local/cuda
-ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
-ENV BOOST_ROOT=/usr/include/boost
-ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
-
-ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
-ARG GPU_APP_COLLECTION_BRANCH=dev
-
-RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
-      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
-      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
-      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
-      python3-doc python3-tk binfmt-support psmisc apt-utils && apt-get clean 
-
-
-# Create and activate a virtual environment, venv is needed because of PEP 668
-RUN python3 -m venv /venv
-ENV PATH="/venv/bin:$PATH"
-RUN pip3 install --upgrade pip
-RUN pip3 install pyyaml plotly psutil
-
-# For CI, only build the CI apps and pull the regression data
-# Clone the gpu-app-collection repository. 
-RUN git clone --recurse-submodules --branch ${GPU_APP_COLLECTION_BRANCH} --single-branch \
-      https://github.com/accel-sim/gpu-app-collection.git
-# Build the CI apps and pull the regression data
-RUN cd gpu-app-collection && bash test-build.sh ci && bash get_regression_data.sh

From ad2b0cb7d1affe7d4f8e2372da8248e28d06c99a Mon Sep 17 00:00:00 2001
From: JRPAN <25518778+JRPan@users.noreply.github.com>
Date: Mon, 2 Nov 2026 14:49:22 -0500
Subject: [PATCH 4/5] Rebuild the gpu-app-collection clone when its dev moves

The clone layer's cache key never changed, so a build reused the clone from
the first run on tgrogers-pc02. That would keep the 13.2 image failing after
accel-sim/gpu-app-collection#91 lands. CI now passes dev's HEAD as a build arg.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/docker-images.yml                | 14 +++++++++++---
 dockerfiles/README.md                              |  6 ++++--
 ...-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile |  2 ++
 .../Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile    |  2 ++
 .../Ubuntu-24.04-cuda-12.8-minimal.Dockerfile      |  2 ++
 dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile      |  2 ++
 .../Ubuntu-24.04-cuda-12.9-minimal.Dockerfile      |  2 ++
 dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile      |  2 ++
 .../Ubuntu-24.04-cuda-13.2-minimal.Dockerfile      |  2 ++
 9 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
index ee32c5887a..ddb6f3c207 100644
--- a/.github/workflows/docker-images.yml
+++ b/.github/workflows/docker-images.yml
@@ -1,6 +1,6 @@
 # Builds the CI images in dockerfiles/ (one <tag>.Dockerfile per ghcr tag).
-# PR: build the changed images only, no push. Merge to dev: build again (a cache hit on
-# tgrogers-pc02, so only the push costs time) and push. Manual: rebuild the named images.
+# PR: build the changed images only, no push. Merge to dev: build again and push (a cache hit on
+# tgrogers-pc02 unless gpu-app-collection's dev moved since the PR built). Manual: rebuild the named images.
 # Which images count as changed: .github/scripts/docker-plan.sh.
 name: Docker images
 
@@ -17,7 +17,7 @@ on:
         required: true
         type: string
       no_cache:
-        description: "Rebuild every layer (e.g. to pick up a newer gpu-app-collection)"
+        description: "Rebuild every layer, including apt and pip"
         type: boolean
         default: false
 
@@ -72,6 +72,13 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Resolve gpu-app-collection dev HEAD
+        id: apps
+        run: |
+          sha=$(git ls-remote https://github.com/accel-sim/gpu-app-collection.git refs/heads/dev | cut -f1)
+          [[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "::error::expected a 40-hex dev SHA, got '$sha'"; exit 1; }
+          echo "sha=$sha" >> "$GITHUB_OUTPUT"
+
       - name: Build${{ github.event_name != 'pull_request' && ' and push' || '' }}
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
@@ -79,6 +86,7 @@ jobs:
           file: dockerfiles/${{ matrix.image }}.Dockerfile
           push: ${{ github.event_name != 'pull_request' }}
           no-cache: ${{ inputs.no_cache == true }}
+          build-args: GPU_APP_COLLECTION_SHA=${{ steps.apps.outputs.sha }}
           tags: ghcr.io/${{ env.IMAGE }}:${{ matrix.image }}
           labels: |
             org.opencontainers.image.source=https://github.com/${{ github.repository }}
diff --git a/dockerfiles/README.md b/dockerfiles/README.md
index 697d300364..6ee51fa923 100644
--- a/dockerfiles/README.md
+++ b/dockerfiles/README.md
@@ -15,8 +15,10 @@ The `*-minimal` images are what CI runs (`IMAGE_TAG` in `.github/workflows/main.
   - A modified file always builds.
   - A new file builds only if its tag is not on ghcr yet.
 - **Merge to `dev`:** the same rules, followed by a push to ghcr.
-  - The build runs on the same docker daemon as the PR's build, so it is a cache hit and the push is the only real work.
-- **Manual:** Actions → Docker images → Run workflow, with the tags to build. Tick `no_cache` to rebuild from scratch, for example to pick up a newer gpu-app-collection.
+  - The build runs on the same docker daemon as the PR's build, so it is a cache hit and the push is the only real work, unless gpu-app-collection's `dev` moved in between.
+- **Manual:** Actions → Docker images → Run workflow, with the tags to build. Tick `no_cache` to rebuild from scratch, including apt and pip.
+
+The workflow passes gpu-app-collection `dev`'s HEAD as `GPU_APP_COLLECTION_SHA`, so a built image always has the current apps. The arg only keys the cache. It does not choose what is cloned.
 
 ## Build host
 
diff --git a/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile b/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile
index 0634f98b25..f2a819e17e 100644
--- a/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile
+++ b/dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile
@@ -48,6 +48,8 @@ RUN git clone https://github.com/riscv-collab/riscv-gnu-toolchain.git \
 && cd .. \
 && rm -rf riscv-gnu-toolchain
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 # Setup GPU app collection in SST mode
 # Also remove prebuilt x86 binaries
 # Also remove all data except for rodinia-2.0
diff --git a/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile b/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile
index 775ea740c2..2a0794ce94 100644
--- a/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile
+++ b/dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile
@@ -28,6 +28,8 @@ ENV PATH="/venv/bin:$PATH"
 RUN pip3 install --upgrade pip
 RUN pip3 install pyyaml plotly psutil
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 # For CI, only build the CI apps and pull the regression data
 # Clone the gpu-app-collection repository
 RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
index bdf0521b97..6092686411 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile
@@ -27,6 +27,8 @@ ENV PATH="/venv/bin:$PATH"
 RUN pip3 install --upgrade pip
 RUN pip3 install pyyaml plotly psutil
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 # For CI, only build the CI apps and pull the regression data
 # Clone the gpu-app-collection repository
 RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile
index cffcdfb82a..f291d6ef7d 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.8.Dockerfile
@@ -26,6 +26,8 @@ ENV PATH="/venv/bin:$PATH"
 RUN pip3 install --upgrade pip
 RUN pip3 install pyyaml plotly psutil
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git && cd gpu-app-collection && bash test-build.sh && bash get_regression_data.sh
 
 
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile
index 34e69d08c5..e088976538 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.9-minimal.Dockerfile
@@ -26,6 +26,8 @@ ENV PATH="/venv/bin:$PATH"
 RUN pip3 install --upgrade pip
 RUN pip3 install pyyaml plotly psutil
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 # For CI, only build the CI apps and pull the regression data
 # Clone the gpu-app-collection repository
 RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
diff --git a/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile
index 226119fa63..8ae8df1e92 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-12.9.Dockerfile
@@ -26,6 +26,8 @@ ENV PATH="/venv/bin:$PATH"
 RUN pip3 install --upgrade pip
 RUN pip3 install pyyaml plotly psutil
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git && cd gpu-app-collection && bash test-build.sh && bash get_regression_data.sh
 
 
diff --git a/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
index 807ffe41ba..78ca67980f 100644
--- a/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
+++ b/dockerfiles/Ubuntu-24.04-cuda-13.2-minimal.Dockerfile
@@ -31,6 +31,8 @@ ENV PATH="/venv/bin:$PATH"
 RUN pip3 install --upgrade pip
 RUN pip3 install pyyaml plotly psutil
 
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
 # For CI, only build the CI apps and pull the regression data
 # Clone the gpu-app-collection repository
 RUN git clone --recurse-submodules --branch ${GPU_APP_COLLECTION_BRANCH} --single-branch \

From d74182f01aa46256726483a4a0ad6ddbf7c2a881 Mon Sep 17 00:00:00 2001
From: JRPAN <25518778+JRPan@users.noreply.github.com>
Date: Mon, 2 Nov 2026 14:53:38 -0500
Subject: [PATCH 5/5] Restore the 13.1.1-minimal image

It only failed because gpu-app-collection dev could not build under CUDA 13,
which accel-sim/gpu-app-collection#91 fixes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 ...buntu-24.04-cuda-13.1.1-minimal.Dockerfile | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile

diff --git a/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile b/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
new file mode 100644
index 0000000000..c97e70b8a0
--- /dev/null
+++ b/dockerfiles/Ubuntu-24.04-cuda-13.1.1-minimal.Dockerfile
@@ -0,0 +1,37 @@
+FROM nvidia/cuda:13.1.1-cudnn-devel-ubuntu24.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 13.1.1 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+ARG GPU_APP_COLLECTION_BRANCH=dev
+
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils && apt-get clean 
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# Set by CI to gpu-app-collection dev's HEAD, so a newer dev invalidates the cached clone.
+ARG GPU_APP_COLLECTION_SHA
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository. 
+RUN git clone --recurse-submodules --branch ${GPU_APP_COLLECTION_BRANCH} --single-branch \
+      https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+RUN cd gpu-app-collection && bash test-build.sh ci && bash get_regression_data.sh