[TransferEngine] Port URMA NUMA affinity mode #26
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Build & Test (Linux)' | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "release/**" | |
| pull_request: | |
| branches: | |
| - "main" | |
| - "release/**" | |
| types: [opened, synchronize, reopened, labeled] | |
| workflow_dispatch: {} | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Build the artifact tested below through the exact release wheel path. The | |
| # Ubuntu jobs remain consumer tests, not an alternate wheel build environment. | |
| build-wheel: | |
| needs: [spell-check, clang-format, check-paths] | |
| if: &run-ci-for-source-changes >- | |
| (needs.check-paths.outputs.should-run-downstream == 'true' || | |
| github.event_name == 'workflow_dispatch') && | |
| (github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.action == 'opened' || | |
| contains(github.event.pull_request.labels.*.name, 'run-ci')) | |
| uses: ./.github/workflows/_build-wheel.yaml | |
| with: | |
| python-versions: '["3.10", "3.12"]' | |
| artifact-prefix: mooncake-wheel-ci | |
| version-override: 0.0.0.dev0 | |
| test-wheel-ubuntu: | |
| needs: [spell-check, clang-format, build-wheel] | |
| if: >- | |
| needs.build-wheel.result == 'success' && | |
| (github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.action == 'opened' || | |
| contains(github.event.pull_request.labels.*.name, 'run-ci')) | |
| strategy: | |
| matrix: | |
| ubuntu-version: [ubuntu-22.04, ubuntu-24.04] | |
| python-version: ['3.10', '3.12'] | |
| runs-on: ${{ matrix.ubuntu-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Generate Python version tag | |
| id: generate_tag_test | |
| run: | | |
| echo "python_version_tag=$(echo ${{ matrix.python-version }} | tr -d '.')" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Download wheel artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: mooncake-wheel-ci-py${{ steps.generate_tag_test.outputs.python_version_tag }} | |
| path: mooncake-wheel/dist | |
| - name: Verify wheel file exists | |
| run: | | |
| ls -la mooncake-wheel/dist/ | |
| if [ ! -f mooncake-wheel/dist/*.whl ]; then | |
| echo "ERROR: No wheel file found in mooncake-wheel/dist/" | |
| exit 1 | |
| fi | |
| shell: bash | |
| - name: Free up disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Install CUDA Toolkit | |
| uses: Jimver/cuda-toolkit@v0.2.24 | |
| with: | |
| cuda: '12.8.1' | |
| linux-local-args: '["--toolkit"]' | |
| method: 'network' | |
| - name: Run installation test script | |
| run: | | |
| bash scripts/test_installation.sh | |
| shell: bash | |
| - name: Run tests with ssd | |
| run: | | |
| # Reserve port 50052 (mooncake_client RPC port) so the kernel never | |
| # auto-allocates it as ephemeral source port for other outbound | |
| # connections in the test suite. Without this, a random Python test | |
| # connection can pick src_port=50052, leave a TIME_WAIT on | |
| # <eth0_ip>:50052 for 60s, and block mooncake_client's bind to | |
| # 0.0.0.0:50052 even with SO_REUSEADDR (Linux only relaxes | |
| # TIME_WAIT+bind conflict for same-IP or loopback). | |
| sudo sysctl -w net.ipv4.ip_local_reserved_ports=50052 | |
| source test_env/bin/activate | |
| MC_STORE_MEMCPY=false TEST_SSD_OFFLOAD_IN_EVICT=true ./scripts/run_tests.sh | |
| rm -rf /tmp/mooncake_test_ssd | |
| deactivate | |
| shell: bash | |
| - name: Start Mooncake Master | |
| run: | | |
| source test_env/bin/activate | |
| mkdir -p /tmp/mooncake_storage | |
| mooncake_master \ | |
| --default_kv_lease_ttl=500 \ | |
| --eviction_high_watermark_ratio=0.95 \ | |
| --cluster_id=ci_test_cluster \ | |
| --port 50051 \ | |
| --enable_http_metadata_server=true & | |
| sleep 3 | |
| shell: bash | |
| - name: Run Python Tensor API Performance Test (CI check) | |
| env: | |
| MOONCAKE_MASTER: "127.0.0.1:50051" | |
| MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata" | |
| MOONCAKE_PROTOCOL: "tcp" | |
| LOCAL_HOSTNAME: "127.0.0.1" | |
| run: | | |
| source test_env/bin/activate | |
| python scripts/test_tensor_api.py -n 1 | |
| shell: bash | |
| - name: Run Python Async API Test (CI check) | |
| env: | |
| MOONCAKE_MASTER: "127.0.0.1:50051" | |
| MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata" | |
| MOONCAKE_PROTOCOL: "tcp" | |
| LOCAL_HOSTNAME: "127.0.0.1" | |
| run: | | |
| source test_env/bin/activate | |
| python scripts/test_async_store.py | |
| shell: bash | |
| - name: Test Mooncake Copy/Move API | |
| env: | |
| MOONCAKE_MASTER: "127.0.0.1:50051" | |
| MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata" | |
| MOONCAKE_PROTOCOL: "tcp" | |
| LOCAL_HOSTNAME: "127.0.0.1" | |
| run: | | |
| source test_env/bin/activate | |
| python scripts/test_copy_move_api.py | |
| shell: bash | |
| - name: Run Python Drain HTTP E2E Test (CI check) | |
| env: | |
| MOONCAKE_MASTER: "127.0.0.1:50051" | |
| MOONCAKE_TE_META_DATA_SERVER: "http://127.0.0.1:8080/metadata" | |
| MOONCAKE_PROTOCOL: "tcp" | |
| LOCAL_HOSTNAME: "127.0.0.1" | |
| run: | | |
| source test_env/bin/activate | |
| export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib | |
| python scripts/test_drain_http_api.py --timeout-sec 90 | |
| shell: bash | |
| - name: Run RPC Communicator Bandwidth Test | |
| run: | | |
| source test_env/bin/activate | |
| python mooncake-transfer-engine/tests/rpc_communicator_test.py server --url 127.0.0.1:9004 --data-size 1 & | |
| SERVER_PID=$! | |
| sleep 5 | |
| timeout 10 python mooncake-transfer-engine/tests/rpc_communicator_test.py client --url 127.0.0.1:9004 --threads 2 --data-size 1 || true | |
| kill $SERVER_PID 2>/dev/null || true | |
| wait $SERVER_PID 2>/dev/null || true | |
| - name: Test Mooncake PyTorch Backend (CPU Only) | |
| env: | |
| MC_FORCE_TCP: "true" | |
| run: | | |
| source test_env/bin/activate | |
| python mooncake-pg/tests/test_pg_collectives.py | |
| shell: bash | |
| - name: Test PyTorch >= 2.13 Single-Buffer Collectives (CPU Only) | |
| env: | |
| MC_FORCE_TCP: "true" | |
| run: | | |
| source test_env/bin/activate | |
| python mooncake-pg/tests/test_pg_collectives.py \ | |
| TestMooncakePGCollectivesCPU.test_all_gather_into_tensor \ | |
| TestMooncakePGCollectivesCPU.test_reduce_scatter_sum | |
| shell: bash | |
| - name: Test Safetensor Functions | |
| run: | | |
| source test_env/bin/activate | |
| pip install safetensors | |
| python -m unittest mooncake-wheel.tests.test_safetensor_functions | |
| shell: bash | |
| build-flags: | |
| needs: [spell-check, clang-format, check-paths] | |
| if: *run-ci-for-source-changes | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| python-version: ['3.10', '3.12'] | |
| env: | |
| CI: "true" | |
| SCCACHE_GHA_ENABLED: "true" | |
| PIP_NO_CACHE_DIR: "1" | |
| CMAKE_RELWITHDEBINFO_FLAGS: "-O2 -DNDEBUG" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Free up disk space | |
| uses: ./.github/actions/free-disk-space | |
| - name: Install CUDA Toolkit | |
| uses: Jimver/cuda-toolkit@v0.2.24 | |
| with: | |
| cuda: '12.8.1' | |
| linux-local-args: '["--toolkit"]' | |
| method: 'network' | |
| sub-packages: '["nvcc", "nvrtc-dev"]' | |
| non-cuda-sub-packages: '["libcusparse-dev", "libcublas-dev", "libcusolver-dev"]' | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Configure sccache | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); | |
| core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update -y | |
| sudo apt install -y ninja-build | |
| sudo bash -x dependencies.sh -y | |
| df -h | |
| shell: bash | |
| - name: Build transfer engine only | |
| run: | | |
| cd mooncake-transfer-engine | |
| mkdir build | |
| cd build | |
| export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH | |
| export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH | |
| cmake -G Ninja .. \ | |
| -DUSE_ETCD=OFF \ | |
| -DUSE_CXL=ON \ | |
| -DUSE_REDIS=ON \ | |
| -DUSE_HTTP=ON \ | |
| -DWITH_METRICS=ON \ | |
| -DBUILD_UNIT_TESTS=ON \ | |
| -DBUILD_EXAMPLES=ON \ | |
| -DENABLE_SCCACHE=ON \ | |
| -DUSE_CUDA=OFF \ | |
| -DUSE_MNNVL=OFF \ | |
| -DUSE_UB=OFF \ | |
| -DCMAKE_EXE_LINKER_FLAGS="-L/usr/local/cuda/lib64/stubs" \ | |
| "-DCMAKE_C_FLAGS_RELWITHDEBINFO=${CMAKE_RELWITHDEBINFO_FLAGS}" \ | |
| "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=${CMAKE_RELWITHDEBINFO_FLAGS}" \ | |
| -DENABLE_DEBUG_SYMBOLS=OFF | |
| cmake --build . | |
| sudo cmake --install . | |
| df -h | |
| cd .. | |
| rm -rf build | |
| df -h | |
| shell: bash | |
| spell-check: | |
| name: Spell Check with Typos | |
| if: &run-ci >- | |
| (github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.action == 'opened' || | |
| contains(github.event.pull_request.labels.*.name, 'run-ci')) | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout Actions Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Spell Check Repo | |
| uses: crate-ci/typos@v1.30.2 | |
| reshard-type-check: | |
| name: Check reshard manifest types | |
| if: *run-ci | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Check canonical reshard contract types | |
| run: | | |
| python -m pip install --disable-pip-version-check pyright==1.1.411 | |
| bash scripts/check_reshard_types.sh | |
| shell: bash | |
| release-gate-tests: | |
| name: Test TestPyPI release gate | |
| if: *run-ci | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Run focused release gate tests | |
| run: | | |
| python -m pip install --disable-pip-version-check \ | |
| packaging pytest==8.3.5 | |
| python -m pytest -q mooncake-wheel/tests/test_testpypi_wheel_gate.py | |
| shell: bash | |
| clang-format: | |
| name: Check code format | |
| if: *run-ci | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout Actions Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Need full history for branch comparison | |
| persist-credentials: false | |
| - name: Install clang-format 20 | |
| run: | | |
| wget https://apt.llvm.org/llvm.sh | |
| chmod +x llvm.sh | |
| sudo ./llvm.sh 20 | |
| sudo apt-get install -y clang-format-20 | |
| - name: Check code format | |
| run: | | |
| # Check script exists and is executable | |
| if [[ ! -x ./scripts/code_format.sh ]]; then | |
| echo "Error: code_format.sh not found or not executable" | |
| exit 1 | |
| fi | |
| # Determine base ref for comparison | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| # For PRs: compare against the target branch | |
| BASE_REF="origin/${{ github.base_ref }}" | |
| else | |
| # For push events: use github.event.before to handle multi-commit pushes | |
| BEFORE_SHA="${{ github.event.before }}" | |
| if [ "${BEFORE_SHA}" == "0000000000000000000000000000000000000000" ]; then | |
| # New branch push, compare against default branch | |
| BASE_REF="origin/${{ github.event.repository.default_branch }}" | |
| else | |
| # Normal push (single or multiple commits) | |
| BASE_REF="${BEFORE_SHA}" | |
| fi | |
| fi | |
| echo "Comparing against: ${BASE_REF}" | |
| ./scripts/code_format.sh --check --changed-lines --base "${BASE_REF}" | |
| shell: bash | |
| docs-check: | |
| name: Check Sphinx docs build | |
| if: *run-ci | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| docs: | |
| - 'docs/**' | |
| - 'docs/requirements-docs.txt' | |
| - name: Set up Python | |
| if: steps.filter.outputs.docs == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install documentation dependencies | |
| if: steps.filter.outputs.docs == 'true' | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r docs/requirements-docs.txt | |
| - name: Build docs with strict mode | |
| if: steps.filter.outputs.docs == 'true' | |
| run: | | |
| cd docs | |
| make html SPHINXOPTS=-W | |
| shell: bash | |
| check-paths: | |
| if: *run-ci | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should-run-downstream: ${{ steps.dispatch-override.outputs.src || steps.filter.outputs.src }} | |
| should-run-tent: ${{ steps.dispatch-override.outputs.tent || steps.filter.outputs.tent }} | |
| should-run-reshard: ${{ steps.dispatch-override.outputs.reshard || steps.filter.outputs.reshard }} | |
| steps: | |
| # workflow_dispatch has no PR/push diff context — skip paths-filter and default to true | |
| - name: Default to true for workflow_dispatch | |
| id: dispatch-override | |
| if: github.event_name == 'workflow_dispatch' | |
| run: | | |
| echo "src=true" >> $GITHUB_OUTPUT | |
| echo "tent=true" >> $GITHUB_OUTPUT | |
| echo "reshard=true" >> $GITHUB_OUTPUT | |
| - uses: actions/checkout@v4 | |
| if: github.event_name != 'workflow_dispatch' | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - uses: dorny/paths-filter@v3 | |
| if: github.event_name != 'workflow_dispatch' | |
| id: filter | |
| with: | |
| filters: | | |
| src: | |
| - 'mooncake-*/**' | |
| - 'extern/**' | |
| - 'CMakeLists.txt' | |
| - 'dependencies.sh' | |
| - 'scripts/**' | |
| - '.github/actions/**' | |
| - '.github/workflows/**' | |
| tent: | |
| - 'mooncake-transfer-engine/**' | |
| - 'mooncake-common/**' | |
| - 'CMakeLists.txt' | |
| - 'dependencies.sh' | |
| - '.github/workflows/ci.yml' | |
| reshard: | |
| - 'mooncake-reshard/**' | |
| - 'mooncake-wheel/pyproject.toml' | |
| - 'requirements.txt' | |
| - 'scripts/check_reshard_types.sh' | |
| - '.github/workflows/ci.yml' | |
| reshard-test: | |
| name: Test Reshard (Python ${{ matrix.python-version }}) | |
| needs: [check-paths] | |
| if: >- | |
| (needs.check-paths.outputs.should-run-reshard == 'true' || | |
| github.event_name == 'workflow_dispatch') && | |
| (github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.action == 'opened' || | |
| contains(github.event.pull_request.labels.*.name, 'run-ci')) | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.10', '3.12'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Reshard test dependencies | |
| run: | | |
| python -m pip install --disable-pip-version-check --upgrade pip | |
| python -m pip install --disable-pip-version-check \ | |
| -r requirements.txt pytest==8.3.5 | |
| - name: Test Reshard | |
| run: | | |
| PYTHONPATH=mooncake-reshard/python \ | |
| python -m pytest -q mooncake-reshard/tests | |
| shell: bash | |
| build-wheel-cu13: | |
| needs: [spell-check, clang-format, check-paths] | |
| if: *run-ci-for-source-changes | |
| uses: ./.github/workflows/_build-wheel.yaml | |
| with: | |
| variant: cuda13 | |
| python-versions: '["3.10", "3.12"]' | |
| artifact-prefix: mooncake-wheel-cu130 | |
| version-override: 0.0.0.dev0 | |
| build-wheel-efa: | |
| needs: [spell-check, clang-format, check-paths] | |
| if: *run-ci-for-source-changes | |
| uses: ./.github/workflows/ci_efa.yml | |
| secrets: inherit | |
| build-wheel-rocm: | |
| needs: [spell-check, clang-format, check-paths] | |
| if: >- | |
| (needs.check-paths.outputs.should-run-downstream == 'true' || | |
| github.event_name == 'workflow_dispatch') && | |
| (github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.action == 'opened' || | |
| contains(github.event.pull_request.labels.*.name, 'run-ci')) | |
| uses: ./.github/workflows/ci_rocm.yml | |
| secrets: inherit | |
| integration-test: | |
| needs: [build-wheel-cu13, check-paths] | |
| if: needs.check-paths.outputs.should-run-downstream == 'true' | |
| uses: ./.github/workflows/integration-test.yml | |
| secrets: inherit | |
| tent-ci: | |
| needs: [spell-check, clang-format, check-paths] | |
| if: >- | |
| (needs.check-paths.outputs.should-run-tent == 'true' || | |
| github.event_name == 'workflow_dispatch') && | |
| (github.event_name == 'push' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event.action == 'opened' || | |
| contains(github.event.pull_request.labels.*.name, 'run-ci')) | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: cuda-on | |
| cmake_flags: '-DUSE_CUDA=ON -DCMAKE_EXE_LINKER_FLAGS=-L/usr/local/cuda/lib64/stubs' | |
| need_cuda: true | |
| metrics_flags: '' | |
| - name: cuda-off | |
| cmake_flags: '-DUSE_CUDA=OFF' | |
| need_cuda: false | |
| metrics_flags: '' | |
| - name: cuda-off-metrics-on | |
| cmake_flags: '-DUSE_CUDA=OFF' | |
| need_cuda: false | |
| metrics_flags: '-DTENT_METRICS_ENABLED=ON' | |
| # Exercise the native UB target with its injected mock adapter. A | |
| # Debug build also catches ODR/link errors hidden by optimization. | |
| - name: ub-mock | |
| cmake_flags: '-DUSE_CUDA=OFF -DUSE_UB=ON -DCMAKE_BUILD_TYPE=Debug' | |
| need_cuda: false | |
| metrics_flags: '' | |
| name: tent-ci (${{ matrix.name }}) | |
| env: | |
| CI: "true" | |
| SCCACHE_GHA_ENABLED: "true" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Free up disk space | |
| if: matrix.need_cuda | |
| uses: ./.github/actions/free-disk-space | |
| - name: Install CUDA Toolkit | |
| if: matrix.need_cuda | |
| uses: Jimver/cuda-toolkit@v0.2.24 | |
| with: | |
| cuda: '12.8.1' | |
| linux-local-args: '["--toolkit"]' | |
| method: 'network' | |
| sub-packages: '["nvcc", "nvrtc-dev"]' | |
| - name: Run sccache-cache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Configure sccache | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); | |
| core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update -y | |
| sudo apt install -y ninja-build | |
| sudo bash -x dependencies.sh -y | |
| df -h | |
| shell: bash | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Configure project with TENT | |
| run: | | |
| mkdir build-tent | |
| cd build-tent | |
| cmake -G Ninja .. -DUSE_TENT=ON -DUSE_HTTP=ON -DENABLE_SCCACHE=ON -DBUILD_UNIT_TESTS=ON -DBUILD_EXAMPLES=ON -DENABLE_DEBUG_SYMBOLS=OFF ${{ matrix.cmake_flags }} ${{ matrix.metrics_flags }} | |
| shell: bash | |
| - name: Build project with TENT | |
| run: | | |
| if [ "${{ matrix.need_cuda }}" = "true" ]; then | |
| export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH | |
| export LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LIBRARY_PATH | |
| fi | |
| cd build-tent | |
| cmake --build . | |
| sudo cmake --install . | |
| shell: bash | |
| # Check the Transfer Engine Rust crate against the freshly built C++ | |
| # libraries. This is a compile/link check, so it is safe to run on every | |
| # matrix leg (including the cuda-on leg, which only verifies that the | |
| # USE_CUDA code paths compile). | |
| - name: Check Transfer Engine Rust bindings, examples, and tests | |
| run: | | |
| if [ -f /usr/local/cuda/lib64/stubs/libcuda.so ] && \ | |
| [ ! -e /usr/local/cuda/lib64/stubs/libcuda.so.1 ]; then | |
| sudo ln -s libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 | |
| fi | |
| if [ "${{ matrix.need_cuda }}" = "true" ]; then | |
| export LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH | |
| fi | |
| cd mooncake-transfer-engine/rust | |
| export MOONCAKE_BUILD_DIR=$GITHUB_WORKSPACE/build-tent | |
| export MOONCAKE_TE_LIB_DIR=$GITHUB_WORKSPACE/build-tent/mooncake-transfer-engine/src | |
| export MOONCAKE_TE_INCLUDE_DIR=$GITHUB_WORKSPACE/mooncake-transfer-engine/include | |
| cargo test --lib | |
| cargo test --examples --tests --no-run | |
| cargo clean | |
| shell: bash | |
| # Only run tests on the cuda-off leg. GitHub runners have no real GPU; | |
| # with USE_CUDA=ON tent's cuda_probe hits the CUDA stub library at | |
| # runtime and drives some dispatch paths past the fake objects the | |
| # unit tests rely on, causing false failures. cuda-on still validates | |
| # that every #ifdef USE_CUDA branch compiles. | |
| - name: Test (TENT) | |
| id: ctest | |
| if: '!matrix.need_cuda' | |
| run: | | |
| cd build-tent | |
| mkdir -p test-results | |
| ctest --test-dir mooncake-transfer-engine/tent/tests -j \ | |
| --output-on-failure \ | |
| --output-junit "$GITHUB_WORKSPACE/build-tent/test-results/ctest.xml" | |
| shell: bash | |
| - name: Preserve TENT CTest diagnostics | |
| if: ${{ always() && steps.ctest.outcome == 'failure' }} | |
| uses: ./.github/actions/ctest-diagnostics | |
| with: | |
| summary-title: TENT CTest failure (${{ matrix.name }}) | |
| artifact-name: ctest-diagnostics-tent-${{ matrix.name }} | |
| junit-report: build-tent/test-results/ctest.xml | |
| failed-tests-log: build-tent/mooncake-transfer-engine/tent/tests/Testing/Temporary/LastTestsFailed.log | |
| last-test-log: build-tent/mooncake-transfer-engine/tent/tests/Testing/Temporary/LastTest.log | |
| - name: Start Metadata Server | |
| if: '!matrix.need_cuda' | |
| run: | | |
| cd mooncake-transfer-engine/example/http-metadata-server-python | |
| pip install aiohttp | |
| python ./bootstrap_server.py & | |
| sleep 2 | |
| shell: bash | |
| - name: Run Transfer Engine Rust smoke test | |
| if: '!matrix.need_cuda' | |
| env: | |
| MOONCAKE_BUILD_DIR: ${{ github.workspace }}/build-tent | |
| MOONCAKE_TE_RUST_LINK_ASAN: "0" | |
| run: ./scripts/ci/run_transfer_engine_rust_smoke.sh | |
| shell: bash | |
| - name: Run sccache stat for check | |
| if: ${{ env.SCCACHE_PATH != '' }} | |
| shell: bash | |
| run: ${SCCACHE_PATH} --show-stats | |
| ci-gate: | |
| name: CI Gate | |
| if: always() | |
| needs: | |
| - spell-check | |
| - clang-format | |
| - docs-check | |
| - reshard-type-check | |
| - release-gate-tests | |
| - reshard-test | |
| - build-wheel | |
| - build-flags | |
| - test-wheel-ubuntu | |
| - build-wheel-cu13 | |
| - build-wheel-efa | |
| - build-wheel-rocm | |
| - tent-ci | |
| - integration-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check required job results | |
| run: | | |
| failing=$(echo "$NEEDS_JSON" | jq -r ' | |
| to_entries[] | | |
| select(.value.result != "success" and .value.result != "skipped") | | |
| "\(.key): \(.value.result)"') | |
| if [ -n "$failing" ]; then | |
| echo "::error::The following jobs failed or were cancelled:" | |
| echo "$failing" | |
| exit 1 | |
| fi | |
| echo "All checks passed or were acceptably skipped." | |
| env: | |
| NEEDS_JSON: ${{ toJSON(needs) }} |