Move the CI Dockerfiles into dockerfiles/ and build them here

.github/scripts/docker-plan.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Decide which dockerfiles/<tag>.Dockerfile images to build. Prints a JSON array of tags on stdout,
+# and one "build/skip <tag>: <reason>" line per image on stderr (and in the job summary).
+#   workflow_dispatch: the tags in $INPUT_IMAGES (space separated).
+#   pull_request / push: Dockerfiles changed between $BASE and $HEAD. A modified file always
+#   builds; an added one builds only if its tag is not on ghcr yet, so importing or renaming
+#   existing images does not rebuild them.
+# Env: EVENT, BASE, HEAD, INPUT_IMAGES, IMAGE (ghcr repository, e.g. accel-sim/accel-sim-framework)
+set -euo pipefail
+
+on_ghcr() {
+  local token code
+  token=$(curl -fsS "https://ghcr.io/token?scope=repository:$IMAGE:pull&service=ghcr.io" |
+    python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')
+  code=$(curl -s -o /dev/null -w '%{http_code}' -I -H "Authorization: Bearer $token" \
+    -H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json' \
+    "https://ghcr.io/v2/$IMAGE/manifests/$1")
+  case $code in
+    200) return 0 ;;
+    404) return 1 ;;
+    *) echo "ghcr manifest check for $1 returned HTTP $code (expected 200 or 404)" >&2; exit 1 ;;
+  esac
+}
+
+tags=()
+say() { echo "$*" >&2; [ -z "${GITHUB_STEP_SUMMARY:-}" ] || echo "- $*" >> "$GITHUB_STEP_SUMMARY"; }
+if [ "$EVENT" = workflow_dispatch ]; then
+  for t in $INPUT_IMAGES; do
+    [ -f "dockerfiles/$t.Dockerfile" ] || { echo "no dockerfiles/$t.Dockerfile" >&2; exit 1; }
+    tags+=("$t"); say "build $t: requested"
+  done
+else
+  [[ $BASE =~ ^0+$ ]] && BASE=$HEAD~1  # first push of a branch has no "before"
+  while read -r status path; do
+    t=$(basename "$path" .Dockerfile)
+    case $status in
+      M) tags+=("$t"); say "build $t: modified" ;;
+      A) if on_ghcr "$t"; then say "skip $t: added, already on ghcr"; else tags+=("$t"); say "build $t: new image"; fi ;;
+      D) say "skip $t: deleted" ;;
+      *) echo "unexpected git status $status for $path" >&2; exit 1 ;;
+    esac
+  done < <(git diff --no-renames --name-status "$BASE" "$HEAD" -- 'dockerfiles/*.Dockerfile')
+fi
+[ ${#tags[@]} -gt 0 ] || say "nothing to build"
+
+printf '%s\n' "${tags[@]}" | python3 -c 'import json,sys; print(json.dumps([l.strip() for l in sys.stdin if l.strip()]))'

.github/workflows/docker-images.yml

@@ -0,0 +1,85 @@
+# Builds the CI images in dockerfiles/ (one <tag>.Dockerfile per ghcr tag).
+# PR: build the changed images only, no push. Merge to dev: build again (a cache hit on
+# tgrogers-pc02, so only the push costs time) and push. Manual: rebuild the named images.
+# Which images count as changed: .github/scripts/docker-plan.sh.
+name: Docker images
+
+on:
+  pull_request:
+    paths: ["dockerfiles/*.Dockerfile"]
+  push:
+    branches: [dev]
+    paths: ["dockerfiles/*.Dockerfile"]
+  workflow_dispatch:
+    inputs:
+      images:
+        description: "Tags to build (dockerfiles/<tag>.Dockerfile), space separated"
+        required: true
+        type: string
+      no_cache:
+        description: "Rebuild every layer (e.g. to pick up a newer gpu-app-collection)"
+        type: boolean
+        default: false
+
+env:
+  IMAGE: accel-sim/accel-sim-framework
+
+jobs:
+  plan:
+    if: github.repository == 'accel-sim/accel-sim-framework'
+    runs-on: tgrogers-raid
+    outputs:
+      images: ${{ steps.plan.outputs.images }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Pick images to build
+        id: plan
+        env:
+          EVENT: ${{ github.event_name }}
+          # A PR checks out its merge commit, whose first parent is the base branch.
+          BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}
+          HEAD: ${{ github.event_name == 'pull_request' && 'HEAD' || github.sha }}
+          INPUT_IMAGES: ${{ inputs.images }}
+        run: echo "images=$(.github/scripts/docker-plan.sh)" >> "$GITHUB_OUTPUT"
+
+  build:
+    needs: plan
+    if: needs.plan.outputs.images != '[]'
+    strategy:
+      fail-fast: false
+      # Builds share pc02 (40 cores, 125 GB); the Dockerfiles use make -j8, so 4 at once is 32 jobs.
+      max-parallel: 4
+      matrix:
+        image: ${{ fromJSON(needs.plan.outputs.images) }}
+    name: ${{ matrix.image }}
+    # tgrogers-raid has no docker daemon: tgrghci's docker CLI + buildx drive tgrogers-pc02's over ssh.
+    runs-on: tgrogers-raid
+    permissions:
+      contents: read
+      packages: write
+    env:
+      DOCKER_HOST: ssh://tgrogers-pc02
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to ghcr.io
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build${{ github.event_name != 'pull_request' && ' and push' || '' }}
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: dockerfiles
+          file: dockerfiles/${{ matrix.image }}.Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          no-cache: ${{ inputs.no_cache == true }}
+          tags: ghcr.io/${{ env.IMAGE }}:${{ matrix.image }}
+          labels: |
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}

dockerfiles/README.md

@@ -0,0 +1,29 @@
+# CI container images
+
+Each `<tag>.Dockerfile` here builds one image, `ghcr.io/accel-sim/accel-sim-framework:<tag>`.
+These files used to live one per branch in `accel-sim/Dockerfile`, which is now retired.
+
+The `*-minimal` images are what CI runs (`IMAGE_TAG` in `.github/workflows/main.yml`). They hold:
+- the build dependencies
+- the gpu-app-collection CI apps and their regression data
+- the AWS CLI and boto3, so the image can also run AWS Batch jobs (`util/aws_batch`) — 12.8-minimal only so far
+
+## When images build
+
+`.github/workflows/docker-images.yml` builds only the Dockerfiles a change touches, and lists each one as build or skip with a reason. The rules are in `.github/scripts/docker-plan.sh`:
+- **Pull request:** builds the images whose Dockerfile changed, without pushing.
+  - A modified file always builds.
+  - A new file builds only if its tag is not on ghcr yet.
+- **Merge to `dev`:** the same rules, followed by a push to ghcr.
+  - The build runs on the same docker daemon as the PR's build, so it is a cache hit and the push is the only real work.
+- **Manual:** Actions → Docker images → Run workflow, with the tags to build. Tick `no_cache` to rebuild from scratch, for example to pick up a newer gpu-app-collection.
+
+## Build host
+
+Builds run on a `tgrogers-raid` runner, which has no docker daemon. Its account (`tgrghci`) has the docker CLI and buildx installed. The workflow sets `DOCKER_HOST=ssh://tgrogers-pc02`, so the build runs on tgrogers-pc02's daemon.
+
+To build by hand on any docker host:
+
+```bash
+docker build -f dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile dockerfiles
+```

dockerfiles/SST-Integration-Ubuntu-22.04-cuda-11.7-llvm-18.1.8-riscv-gnu-2024.08.06-nightly.Dockerfile

@@ -0,0 +1,67 @@
+FROM ubuntu:22.04
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+ADD . /accel-sim
+
+ENV CUDA_INSTALL_PATH /usr/local/cuda-11.7
+ENV PTXAS_CUDA_INSTALL_PATH /usr/local/cuda-11.7
+ENV GPUAPPS_ROOT /accel-sim/gpu-app-collection
+ENV LLVM_INSTALL_PATH /accel-sim/llvm-install
+ENV RISCV_TOOLCHAIN_INSTALL_PATH /accel-sim/riscv-gnu-install
+
+# Install CUDA
+RUN apt-get update \
+&& apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python-setuptools python3-pip cmake \
+&& apt-get clean \
+&& pip3 install pyyaml plotly psutil \
+&& wget https://developer.download.nvidia.com/compute/cuda/11.7.0/local_installers/cuda_11.7.0_515.43.04_linux.run \
+&& sh cuda_11.7.0_515.43.04_linux.run --silent --toolkit \
+&& rm cuda_11.7.0_515.43.04_linux.run \
+&& rm -rf /usr/local/cuda-11.7/nsight-compute-2022.2.0 \
+&& rm -rf /usr/local/cuda-11.7/nsight-systems-2022.1.3
+
+# Build LLVM 18.1.8
+RUN git clone https://github.com/llvm/llvm-project.git \
+&& mkdir llvm-install \
+&& cd llvm-project \
+&& git checkout llvmorg-18.1.8 \
+&& mkdir build && cd build \
+&& cmake -DLLVM_TARGETS_TO_BUILD="RISCV;X86;NVPTX" -DLLVM_DEFAULT_TARGET_TRIPLE=riscv64-unknown-linux-gnu \
+         -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;lld" -DCMAKE_INSTALL_PREFIX=$LLVM_INSTALL_PATH ../llvm \
+&& cmake --build . -j4 \
+&& cmake --build . --target install \
+&& cd .. && cd .. \
+&& rm -rf llvm-project
+
+# Build RISCV GNU Toolchain 2024.08.06.nightly
+RUN apt-get update && apt-get -y install autoconf automake autotools-dev curl python3 python3-pip libmpc-dev libmpfr-dev libgmp-dev gawk build-essential bison flex texinfo gperf libtool patchutils bc zlib1g-dev libexpat-dev ninja-build git cmake libglib2.0-dev libslirp-dev
+RUN git clone https://github.com/riscv-collab/riscv-gnu-toolchain.git \
+&& mkdir riscv-gnu-install \
+&& cd riscv-gnu-toolchain \
+&& git checkout 2024.08.06 \
+&& ./configure --prefix=$RISCV_TOOLCHAIN_INSTALL_PATH \
+&& make linux -j4 \
+&& cd .. \
+&& rm -rf riscv-gnu-toolchain
+
+# Setup GPU app collection in SST mode
+# Also remove prebuilt x86 binaries
+# Also remove all data except for rodinia-2.0
+RUN export PATH=$CUDA_INSTALL_PATH/bin:$PATH \
+&& git clone https://github.com/accel-sim/gpu-app-collection \
+&& cd gpu-app-collection \
+&& git pull \
+&& git checkout sst_support \
+&& source ./src/setup_environment sst \
+&& make -j -C ./src data \
+&& mv ./data_dirs/cuda/rodinia/2.0-ft . \
+&& rm -rf ./data_dirs \
+&& mkdir -p ./data_dirs/cuda/rodinia/ \
+&& mv ./2.0-ft ./data_dirs/cuda/rodinia/ \
+&& ls ./data_dirs/cuda/rodinia/ \
+&& rm gpucomputingsdk_4.2.9_linux.run \
+&& rm -rf 4.2

dockerfiles/Ubuntu-22.04-cuda-11.7.1-minimal.Dockerfile

@@ -0,0 +1,35 @@
+FROM nvidia/cuda:11.7.1-cudnn8-devel-ubuntu22.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 22.04 with CUDA 11.7.1 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+ENV LD_PRELOAD=/lib/x86_64-linux-gnu/libz.so.1
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils && apt-get clean 
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+RUN cd gpu-app-collection && bash test-build.sh ci && bash get_regression_data.sh

dockerfiles/Ubuntu-24.04-cuda-12.8-minimal.Dockerfile

@@ -0,0 +1,46 @@
+FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu24.04
+LABEL org.opencontainers.image.source=https://github.com/accel-sim/accel-sim-framework
+LABEL org.opencontainers.image.description="Accel-Sim container with Ubuntu 24.04 with CUDA 12.8.0 and CUDNN, for CI runs only"
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+RUN apt-get update && apt-get install -y wget build-essential xutils-dev bison zlib1g-dev flex \
+      libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev git g++ \
+      libxml2-dev vim python3-setuptools python3-pip python3-venv cmake \
+      libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+      python3-doc python3-tk binfmt-support psmisc apt-utils \
+      libzstd-dev && apt-get clean
+
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+# For CI, only build the CI apps and pull the regression data
+# Clone the gpu-app-collection repository
+RUN git clone --recurse-submodules https://github.com/accel-sim/gpu-app-collection.git
+# Build the CI apps and pull the regression data
+# Invoke make directly instead of test-build.sh, which hardcodes a serial
+# `make -C src/ ci`. src/setup_environment is still required: it exports
+# BINDIR/BINSUBDIR/SETENV and the per-CUDA-version gencode flags the
+# suite makefiles read. -j8 propagates to every recursive $(MAKE),
+# including cutlass_mini's cmake-generated makefiles.
+RUN cd gpu-app-collection && source ./src/setup_environment && make -j8 -C src/ ci && bash get_regression_data.sh
+
+# AWS CLI v2 and boto3, so this image can run as an AWS Batch job (accel-sim-framework util/aws_batch)
+RUN apt-get update && apt-get install -y --no-install-recommends unzip && apt-get clean && \
+    wget -qO /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.39.2.zip && \
+    unzip -q /tmp/awscliv2.zip -d /tmp && /tmp/aws/install && rm -rf /tmp/aws /tmp/awscliv2.zip && \
+    aws --version
+RUN pip3 install boto3==1.43.122

dockerfiles/Ubuntu-24.04-cuda-12.8-vllm.Dockerfile

@@ -0,0 +1,56 @@
+FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu24.04
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /accel-sim
+
+ENV CUDA_INSTALL_PATH=/usr/local/cuda
+ENV PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda
+ENV BOOST_ROOT=/usr/include/boost
+ENV PATH=$CUDA_INSTALL_PATH/bin:$PATH
+
+ENV GPUAPPS_ROOT=/accel-sim/gpu-app-collection
+
+# NOTE: `bc` and `libzstd-dev` added to the upstream list -- both are load-bearing
+# for the tracer build, which upstream does at runtime from the mounted source:
+#   bc          -> tracer_tool/Makefile shells out to it for the nvcc>=11.7 ARCH check
+#   libzstd-dev -> tracer_tool/traces-processing links -lzstd
+RUN apt-get update && apt-get install -y \
+    wget build-essential xutils-dev bison zlib1g-dev flex \
+    libglu1-mesa-dev git g++ libssl-dev libxml2-dev libboost-all-dev \
+    vim python3-setuptools python3-pip python3-venv cmake \
+    libfreeimage3 libfreeimage-dev freeglut3-dev pkg-config \
+    python3-doc python3-tk python3.12-venv python3.12-doc \
+    binfmt-support psmisc apt-utils gdb curl bash-completion \
+    bc libzstd-dev && \
+    apt-get clean
+
+# Create and activate a virtual environment, venv is needed because of PEP 668
+RUN python3 -m venv /venv
+ENV PATH="/venv/bin:$PATH"
+RUN pip3 install --upgrade pip
+RUN pip3 install pyyaml plotly psutil
+
+#get Nsys
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update --allow-insecure-repositories && apt update && apt install -y --no-install-recommends gnupg wget \
+    && mkdir -p /etc/apt/keyrings
+RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64/7fa2af80.pub | tee /etc/apt/keyrings/nvidia.asc
+RUN echo "deb [signed-by=/etc/apt/keyrings/nvidia.asc] http://developer.download.nvidia.com/devtools/repos/ubuntu2404/amd64 /" | tee /etc/apt/sources.list.d/nvidia.list
+RUN apt-get update --allow-insecure-repositories
+RUN apt install  -y nsight-systems-cli --allow-unauthenticated
+
+# ---------------------------------------------------------------------------
+# vLLM + LLM tracing stack.
+#
+# Everything above is the upstream accel-sim Dockerfile unchanged (bar the two
+# packages noted). Nothing below changes the upstream workflow: no source is
+# baked in, WORKDIR stays /accel-sim, and there is no ENTRYPOINT override -- the
+# nvidia/cuda base entrypoint already does `exec "$@"`, so you pass a command in
+# and the mounted script builds the tracer, exactly as .github/scripts do.
+#
+# This is the single biggest layer in the image (~10 GB).
+# ---------------------------------------------------------------------------
+RUN pip3 install --no-cache-dir vllm transformers nvtx
+RUN pip3 install --no-cache-dir monai diffusers accelerate