diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cd606208..2c3759f0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -570,6 +570,18 @@ jobs: uses: ./.github/actions/ios-gamebuild-test with: shell: msys2 {0} + # (the game code generator's unicorn, as lasgen is built: an x64 python.org Python; last, so its PATH + # change touches no step above) + - uses: actions/setup-python@v5 + id: uc-py + with: + python-version: '3.12' + architecture: x64 + - name: unicorn's longjmp in the generator (#779) + shell: pwsh + run: | + & '${{ steps.uc-py.outputs.python-path }}' -m pip install -q unicorn==2.1.4 + & '${{ steps.uc-py.outputs.python-path }}' tools/test_win_longjmp.py # Windows on ARM (experimental builds): the same app build and Windows tests natively on an ARM64 runner, MSYS2 # CLANGARM64 as tools/setup_windows.ps1 sets it up there. Left out: the x86 SIMD rounding test (-mfma) and the shader @@ -653,6 +665,18 @@ jobs: uses: ./.github/actions/ios-gamebuild-test with: shell: msys2 {0} + # (the game code generator's unicorn, as lasgen is built: an x64 python.org Python, under x64 emulation here; last, so its PATH + # change touches no step above) + - uses: actions/setup-python@v5 + id: uc-py + with: + python-version: '3.12' + architecture: x64 + - name: unicorn's longjmp in the generator (#779) + shell: pwsh + run: | + & '${{ steps.uc-py.outputs.python-path }}' -m pip install -q unicorn==2.1.4 + & '${{ steps.uc-py.outputs.python-path }}' tools/test_win_longjmp.py # Android (arm64): the app (android/, libmain.so with the moving guest window) builds with the NDK, and the APK holds # only the runtime, SDL, shaderc and the shipped fonts and icon - nothing of the game, whose code is built on the diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml deleted file mode 100644 index 06a3d0e00..000000000 --- a/.github/workflows/experimental.yml +++ /dev/null @@ -1,261 +0,0 @@ -# Experimental builds, a rolling pre-release for experimental platforms: ones that aren't official but are available -# for testing (Linux aarch64, Windows on ARM; Android was one until it joined the releases in Beta 6: release.yml). From 1.0.0 they are never on the normal releases. Each build goes on the rolling pre-release "experimental", replacing that platform's previous file, -# or, for a reporter testing a fix, on a test build of its own. None of the files holds game code or game files. -# Run by pushing a tag at the commit to build (from any branch that has this file): -# experimental- -> the rolling "experimental" release; the pushed tag is deleted after the upload -# test-arm- -> a pre-release "test-arm-" (start its notes with the test build warning after) -# or, once this file is on main: gh workflow run experimental.yml -f ref= [-f test=] -# The build's version is set here, not in runtime/ui/launcher.mm: experimental-- (or the -# test-arm- tag), with LARECOMP_EXPERIMENTAL_STAMP = that time, so its update check only offers a newer -# experimental build of the same platform (runtime/ui/update_tags.h, newest_experimental). -name: Experimental -on: - push: - tags: ['experimental-*', 'test-arm-*'] - workflow_dispatch: - inputs: - ref: - description: 'Branch or commit to build' - required: true - test: - description: 'Test build name (test-arm-); empty: the rolling experimental release' - required: false - default: '' -permissions: - contents: read -concurrency: # (one upload to the rolling release at a time) - group: experimental-${{ startsWith(github.ref_name, 'test-arm-') && github.ref_name || inputs.test || 'rolling' }} - cancel-in-progress: false - -jobs: - check: - runs-on: ubuntu-24.04 - outputs: - sha: ${{ steps.v.outputs.sha }} - version: ${{ steps.v.outputs.version }} - stamp: ${{ steps.v.outputs.stamp }} - release: ${{ steps.v.outputs.release }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.sha }} - - name: Version - id: v - env: - EVENT: ${{ github.event_name }} - PUSHED: ${{ github.ref_name }} - TEST: ${{ inputs.test }} - run: | - SHA=$(git rev-parse HEAD) - STAMP=$(date -u +%Y%m%d%H%M) - if [ "$EVENT" = push ]; then - case $PUSHED in test-arm-*) TEST=${PUSHED#test-arm-} ;; *) TEST= ;; esac - fi - if [ -n "$TEST" ]; then - [[ $TEST =~ ^[A-Za-z0-9._-]+$ ]] || { echo "::error::test name '$TEST': letters, digits, . _ - only"; exit 1; } - VERSION=test-arm-$TEST; RELEASE=$VERSION - else - VERSION=experimental-$STAMP-${SHA::7}; RELEASE=experimental - fi - { echo "sha=$SHA"; echo "version=$VERSION"; echo "stamp=$STAMP"; echo "release=$RELEASE"; } >> "$GITHUB_OUTPUT" - echo "$VERSION ($RELEASE): $SHA" - - # the same build as release.yml's Linux job, on an arm64 runner - linux: - needs: check - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - include: - - { arch: aarch64, runner: ubuntu-24.04-arm } - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.check.outputs.sha }} - - name: Experimental version - env: - VERSION: ${{ needs.check.outputs.version }} - STAMP: ${{ needs.check.outputs.stamp }} - run: | - F=runtime/ui/launcher.mm - sed -i "s/^#define LARECOMP_VERSION \".*\"/#define LARECOMP_VERSION \"$VERSION\"/; s/^#define LARECOMP_EXPERIMENTAL_STAMP .*/#define LARECOMP_EXPERIMENTAL_STAMP ${STAMP}LL/" $F - grep -q "^#define LARECOMP_VERSION \"$VERSION\"" $F && grep -q "^#define LARECOMP_EXPERIMENTAL_STAMP ${STAMP}LL" $F || - { echo "::error::$F: LARECOMP_VERSION / LARECOMP_EXPERIMENTAL_STAMP not set"; exit 1; } - - name: Deps cache - id: deps - uses: actions/cache/restore@v4 - with: - path: build/linux/${{ matrix.arch }}/deps - key: deps-linux-${{ matrix.arch }}-${{ hashFiles('tools/linux/build_deps.sh', 'tools/linux/Containerfile') }} - - name: Wheels cache - id: wheels - uses: actions/cache/restore@v4 - with: - path: build/linux/${{ matrix.arch }}/wheels - key: wheels-linux-${{ matrix.arch }}-${{ hashFiles('third_party/LAS-Randomizer/requirements.txt', 'tools/linux/rando_helper.sh', 'tools/linux/Containerfile') }} - - name: AppImage - run: make appimage ARCH=${{ matrix.arch }} - - uses: actions/cache/save@v4 - if: always() && steps.deps.outputs.cache-hit != 'true' && hashFiles(format('build/linux/{0}/deps/lib/libshaderc_shared.so*', matrix.arch)) != '' - with: - path: build/linux/${{ matrix.arch }}/deps - key: ${{ steps.deps.outputs.cache-primary-key }} - - uses: actions/cache/save@v4 - if: always() && steps.wheels.outputs.cache-hit != 'true' && hashFiles(format('build/linux/{0}/rando-venv/.ok', matrix.arch)) != '' - with: - path: build/linux/${{ matrix.arch }}/wheels - key: ${{ steps.wheels.outputs.cache-primary-key }} - - name: No game code - run: | - A=build/linux/${{ matrix.arch }}/AppDir - bad=$(find "$A" -type d \( -name extracted -o -name 'generated*' -o -name exefs -o -name romfs \); - find "$A" -name '*.c' ! -path '*/zig/lib/*' ! -path '*/gamelib-src/gamelib_entry.c'; find "$A" -name 'lasgame*') - [ -z "$bad" ] || { echo "::error::must not ship: $bad"; exit 1; } - find "$A" -type f \( -perm /111 -o -name '*.so*' \) | while read -r f; do - if readelf -Ws --wide "$f" 2>/dev/null | grep ' f_70' >/dev/null; then echo "::error::game code in $f"; exit 1; fi - done - ls -la build/dist - - uses: actions/upload-artifact@v4 - with: - name: linux-${{ matrix.arch }} - path: build/dist/*.AppImage - if-no-files-found: error - compression-level: 0 - - # the same build as release.yml's Windows job, natively on an arm64 runner (MSYS2 CLANGARM64); the generator - # helpers are built with an x64 Python and run under emulation (capstone / unicorn have no ARM64 Windows wheels) - windows: - needs: check - runs-on: windows-11-arm - defaults: - run: - shell: msys2 {0} - steps: - - name: LF line endings, as tools/setup_windows.ps1 sets for clones - shell: pwsh - run: git config --global core.autocrlf false - - uses: actions/checkout@v4 - with: - ref: ${{ needs.check.outputs.sha }} - - uses: actions/setup-python@v5 - id: py - with: - python-version: '3.12' - architecture: x64 - - uses: msys2/setup-msys2@v2 - with: - msystem: CLANGARM64 - update: true - install: make git curl tar xz diffutils zip unzip - pacboy: clang:p lld:p sdl3:p vulkan-devel:p shaderc:p spirv-tools:p pkgconf:p vulkan-memory-allocator:p - - name: Experimental version - env: - VERSION: ${{ needs.check.outputs.version }} - STAMP: ${{ needs.check.outputs.stamp }} - run: | - F=runtime/ui/launcher.mm - sed -i "s/^#define LARECOMP_VERSION \".*\"/#define LARECOMP_VERSION \"$VERSION\"/; s/^#define LARECOMP_EXPERIMENTAL_STAMP .*/#define LARECOMP_EXPERIMENTAL_STAMP ${STAMP}LL/" $F - grep -q "^#define LARECOMP_VERSION \"$VERSION\"" $F && grep -q "^#define LARECOMP_EXPERIMENTAL_STAMP ${STAMP}LL" $F || - { echo "::error::$F: LARECOMP_VERSION / LARECOMP_EXPERIMENTAL_STAMP not set"; exit 1; } - - name: Deps cache - id: deps - uses: actions/cache/restore@v4 - with: - path: | - build/deps-win - !build/deps-win/src - key: deps-windows-arm64-${{ hashFiles('tools/build_deps_windows.sh') }} - - name: Deps (LGPL FFmpeg) - run: make deps-windows - - uses: actions/cache/save@v4 - if: steps.deps.outputs.cache-hit != 'true' - with: - path: | - build/deps-win - !build/deps-win/src - key: ${{ steps.deps.outputs.cache-primary-key }} - - name: Generator helpers - run: | - PY=$(cygpath -u '${{ steps.py.outputs.python-path }}') - make rando-helper RANDO_PY="$PY" && make gen-helper RANDO_PY="$PY" - - name: Compiler for the player's machine - run: tools/fetch_zig.sh - - name: Zip - run: make -j"$(nproc)" dist - - name: No game files or generated C - run: | - D="build/dist/LAS Recomp" - bad=$(find "$D" -type d \( -name extracted -o -name 'generated*' -o -name exefs -o -name romfs \); - find "$D" -name '*.c' ! -path '*/zig/lib/*' ! -path '*/gamelib-src/gamelib_entry.c'; find "$D" -name 'lasgame*') - [ -z "$bad" ] || { echo "::error::must not ship: $bad"; exit 1; } - ls -la build/dist - - uses: actions/upload-artifact@v4 - with: - name: windows-arm64 - path: build/dist/*.zip - if-no-files-found: error - compression-level: 0 - - publish: - needs: [check, linux, windows] - # (a platform whose build failed keeps its previous file; the others still go up) - if: always() && needs.check.result == 'success' && (needs.linux.result == 'success' || needs.windows.result == 'success') - runs-on: ubuntu-24.04 - permissions: - contents: write # (only here: the experimental / test-arm release) - steps: - - uses: actions/download-artifact@v4 - with: - path: dist - merge-multiple: true - - name: Upload - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - RELEASE: ${{ needs.check.outputs.release }} - VERSION: ${{ needs.check.outputs.version }} - SHA: ${{ needs.check.outputs.sha }} - EVENT: ${{ github.event_name }} - PUSHED: ${{ github.ref_name }} - run: | - ls -la dist - URL=https://github.com/$GH_REPO - # (the rolling release's notes are written on every run, so a platform added here shows up on its page) - if [ "$RELEASE" = experimental ]; then - TITLE="Experimental builds" - printf '%s\n' \ - "> [!WARNING]" \ - "> **These builds are experimental.** They are for platforms that aren't official but are available for testing, so expect problems. For everything else, use the **[latest release]($URL/releases/latest)**." \ - "" \ - "The latest build for each platform is below. Each new build replaces the old one, and the game tells you on its start screen when a newer one is here." \ - "" \ - "- Linux aarch64 (ARM): \`linux-aarch64.AppImage\`" \ - "- Windows on ARM: \`windows-arm64.zip\` (Windows 11 on ARM, with a Vulkan 1.3 graphics driver)" \ - "" \ - "## Help us make these better" \ - "" \ - "We really want to hear how these builds run on your device, good or bad. We don't have most of this hardware ourselves, so your reports are how these platforms get better. When something doesn't work, we'll work through it with you on the issue, make test builds for you to try, and fix what we can." \ - "" \ - "To report, open an issue on the [issues page]($URL/issues) with your device, its graphics driver, screenshots of anything that looks wrong, and the log (\`lasrecomp.log\` in the data folder)." > notes.md - else - TITLE="Test build: ${RELEASE#test-}" - echo "ARM test build from $SHA." > notes.md - fi - if ! gh release view "$RELEASE" >/dev/null 2>&1; then - gh release create "$RELEASE" --prerelease --title "$TITLE" --notes-file notes.md --target "$SHA" - elif [ "$RELEASE" = experimental ]; then - gh release edit "$RELEASE" --notes-file notes.md - fi - # this platform's previous file(s) out, the new one in - for f in dist/*; do - plat=${f##*/}; plat=${plat#*"$VERSION"-} - gh release view "$RELEASE" --json assets -q '.assets[].name' | while read -r old; do - case $old in *"-$plat") [ "$old" = "$(basename "$f" | tr ' ' .)" ] || gh release delete-asset "$RELEASE" "$old" -y ;; esac - done - gh release upload "$RELEASE" "$f" --clobber - done - # the rolling release's tag follows its newest build (its page shows that commit) - if [ "$RELEASE" = experimental ]; then gh api -X PATCH "repos/$GH_REPO/git/refs/tags/experimental" -f sha="$SHA" -F force=true >/dev/null; fi - # (the pushed experimental-* tag only started this run) - if [ "$EVENT" = push ] && [ "$RELEASE" = experimental ]; then gh api -X DELETE "repos/$GH_REPO/git/refs/tags/$PUSHED" || true; fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40dd5e6c5..a640a0155 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,13 +12,15 @@ # platforms: test builds (test-*, ci-test-*) only - the platforms the fix targets and wants feedback from, e.g. # -f platforms=windows or -f platforms=macos,linux: every platform is still built (a compile check), only those # files go on the test build. Releases always ship every platform. +# windows-arm64 (Windows on ARM) is for test builds only and only built when named: it isn't a release platform +# (it ships on the nightly page, nightly.yml), e.g. -f platforms=windows-arm64,linux-aarch64. # Dry run of this workflow itself: a tag starting with ci-test- may differ from LARECOMP_VERSION (the files keep its # name); its draft is a prerelease, as for test- builds. # Also on a release published on GitHub (its tag = LARECOMP_VERSION at the tagged commit): the files are built and # attached to it, with a "being built" line on top of its notes meanwhile and the sha256 table under them after. A # release that already has files (a draft this workflow made, then published) is left alone. # Linux aarch64 stays on these releases through Beta 6 (so players on older aarch64 builds still get offered an update); -# from 1.0.0 it is only on the rolling "experimental" pre-release (experimental.yml). +# from 1.0.0 it is only on the nightly page (nightly.yml). name: Release on: release: @@ -33,7 +35,7 @@ on: required: false default: '' platforms: - description: 'Test builds only: the platforms the fix targets (macos, windows, linux, linux-aarch64, android, ios; comma list). Empty: all' + description: 'Test builds only: the platforms the fix targets (macos, windows, linux, linux-aarch64, android, ios, windows-arm64; comma list). Empty: all but windows-arm64' required: false default: '' permissions: @@ -50,6 +52,7 @@ jobs: version: ${{ steps.v.outputs.version }} skip: ${{ steps.v.outputs.skip }} platforms: ${{ steps.v.outputs.platforms }} + arm64: ${{ steps.v.outputs.arm64 }} steps: - uses: actions/checkout@v4 with: @@ -65,8 +68,8 @@ jobs: run: | # (a published release that already has its files: made by a dispatched run - nothing to do) if [ "$(jq length <<< "$ASSETS")" != 0 ]; then echo "skip=true" >> "$GITHUB_OUTPUT"; echo "$TAG already has files"; exit 0; fi - # (experimental.yml's own releases) - if [[ $TAG == experimental* || $TAG == test-arm-* ]]; then echo "skip=true" >> "$GITHUB_OUTPUT"; echo "$TAG: experimental.yml"; exit 0; fi + # (the old rolling experimental release and test-arm-* test builds, from the retired experimental.yml) + if [[ $TAG == experimental* || $TAG == test-arm-* ]]; then echo "skip=true" >> "$GITHUB_OUTPUT"; echo "$TAG: not a release.yml build"; exit 0; fi [[ $TAG =~ ^[A-Za-z0-9._-]+$ ]] || { echo "::error::tag '$TAG': letters, digits, . _ - only"; exit 1; } SHA=$(git rev-parse HEAD) VERSION=$(sed -n 's/^#define LARECOMP_VERSION "\(.*\)"/\1/p' runtime/ui/launcher.mm) @@ -87,8 +90,11 @@ jobs: if [ -z "${PLATFORMS// /}" ]; then PLATFORMS="macos windows linux linux-aarch64 android ios" elif [[ $TAG != test-* && $TAG != ci-test-* ]]; then echo "::error::platforms: only for test builds; a release ships all"; exit 1; fi for p in $PLATFORMS; do - case $p in macos|windows|linux|linux-aarch64|android|ios) ;; *) echo "::error::platform '$p': macos, windows, linux, linux-aarch64, android or ios"; exit 1 ;; esac + case $p in macos|windows|linux|linux-aarch64|android|ios|windows-arm64) ;; + *) echo "::error::platform '$p': macos, windows, linux, linux-aarch64, android, ios or windows-arm64"; exit 1 ;; esac done + # (Windows on ARM: built only for a test build that names it) + if [[ " $PLATFORMS " == *" windows-arm64 "* ]]; then echo "arm64=true" >> "$GITHUB_OUTPUT"; fi echo "platforms=$(echo $PLATFORMS)" >> "$GITHUB_OUTPUT" echo "sha=$SHA" >> "$GITHUB_OUTPUT" echo "version=$VERSION" >> "$GITHUB_OUTPUT" @@ -226,6 +232,74 @@ jobs: if-no-files-found: error compression-level: 0 + # Windows on ARM (MSYS2 CLANGARM64): only for a test build that names windows-arm64 (it ships on the nightly page) + windows-arm64: + needs: check + if: needs.check.outputs.skip != 'true' && needs.check.outputs.arm64 == 'true' + runs-on: windows-11-arm + defaults: + run: + shell: msys2 {0} + steps: + - name: LF line endings, as tools/setup_windows.ps1 sets for clones + shell: pwsh + run: git config --global core.autocrlf false + - uses: actions/checkout@v4 + with: + ref: ${{ needs.check.outputs.sha }} + # the generator helpers are frozen from a python.org-style CPython 3.12 (MSYS2's Python has no oead wheel) + - uses: actions/setup-python@v5 + id: py + with: + python-version: '3.12' + architecture: x64 # (as nightly.yml: the x64 Python, emulated, for the generator helpers) + # MSYS2 CLANGARM64, as nightly.yml's Windows on ARM build + - uses: msys2/setup-msys2@v2 + with: + msystem: CLANGARM64 + update: true + install: make git curl tar xz diffutils zip unzip + pacboy: clang:p lld:p sdl3:p vulkan-devel:p shaderc:p spirv-tools:p pkgconf:p vulkan-memory-allocator:p + - name: Deps cache + id: deps + uses: actions/cache/restore@v4 + with: + path: | + build/deps-win + !build/deps-win/src + key: deps-windows-arm64-${{ hashFiles('tools/build_deps_windows.sh') }} + - name: Deps (LGPL FFmpeg) + run: make deps-windows + - uses: actions/cache/save@v4 + if: steps.deps.outputs.cache-hit != 'true' + with: + path: | + build/deps-win + !build/deps-win/src + key: ${{ steps.deps.outputs.cache-primary-key }} + - name: Generator helpers + run: | + PY=$(cygpath -u '${{ steps.py.outputs.python-path }}') + make rando-helper RANDO_PY="$PY" && make gen-helper RANDO_PY="$PY" + - name: Compiler for the player's machine + run: tools/fetch_zig.sh + # (make dist: the exe holds no game code, no GPL codec libraries) + - name: Release zip + run: make -j"$(nproc)" dist + - name: No game files or generated C + run: | + D="build/dist/LAS Recomp" + bad=$(find "$D" -type d \( -name extracted -o -name 'generated*' -o -name exefs -o -name romfs \); + find "$D" -name '*.c' ! -path '*/zig/lib/*' ! -path '*/gamelib-src/gamelib_entry.c'; find "$D" -name 'lasgame*') + [ -z "$bad" ] || { echo "::error::must not ship: $bad"; exit 1; } + ls -la build/dist + - uses: actions/upload-artifact@v4 + with: + name: windows-arm64 + path: build/dist/*.zip + if-no-files-found: error + compression-level: 0 + # The iOS kit (tools/ios_kit.sh, one .laskit file): the app without game code (make ios-skeleton: Xcode's iOS SDK, # so a Mac), the Mach-O linker for each desktop (tools/fetch_lld.sh, all five targets) and the compiler's Darwin # headers (from the zig archive tools/fetch_zig.sh fetched: the same files for every host). The start screen on @@ -373,7 +447,7 @@ jobs: compression-level: 0 # (a release published on GitHub: a line on top of its notes while the files build) - # the Android app (as experimental.yml's android job) as a release build: this release's LARECOMP_VERSION, no + # the Android app as a release build: this release's LARECOMP_VERSION, no # experimental stamp (its update check takes the normal releases), signed with the published APKs' key (the # repository's secrets ANDROID_KEYSTORE_B64 / ANDROID_KEYSTORE_PASSWORD: it installs over an experimental build) android: @@ -415,7 +489,7 @@ jobs: [ -n "$KEYSTORE_B64" ] && [ -n "$LARECOMP_KEYSTORE_PASSWORD" ] || { echo "::error::the signing key secrets are not set"; exit 1; } export LARECOMP_KEYSTORE=$RUNNER_TEMP/lasrecomp.keystore echo "$KEYSTORE_B64" | base64 -d > "$LARECOMP_KEYSTORE" - # (Android's version code only grows: minutes since 2026-01-01, as experimental.yml's) + # (Android's version code only grows: minutes since 2026-01-01) export LARECOMP_VERSION_CODE=$(( ($(date -u +%s) - $(date -u -d 20260101 +%s)) / 60 )) (cd android && ./gradlew assembleRelease --console=plain) rm -f "$LARECOMP_KEYSTORE" @@ -452,8 +526,9 @@ jobs: gh release edit "$TAG" --notes-file new.md release: - needs: [check, macos, windows, linux, android, ios] - if: needs.check.outputs.skip != 'true' + needs: [check, macos, windows, windows-arm64, linux, android, ios] + # (windows-arm64 is skipped unless a test build names it: a skipped job doesn't hold the draft back, a failed one does) + if: always() && needs.check.result == 'success' && needs.check.outputs.skip != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') runs-on: ubuntu-24.04 permissions: contents: write # (only here: the draft release) @@ -468,16 +543,19 @@ jobs: EVENT: ${{ github.event_name }} SHA: ${{ needs.check.outputs.sha }} PLATFORMS: ${{ needs.check.outputs.platforms }} + ARM64: ${{ needs.check.outputs.arm64 }} GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} run: | cd dist ls -la - [ "$(ls | wc -l)" = 6 ] || { echo "::error::expected 6 files"; exit 1; } + N=6; [ "$ARM64" = true ] && N=7 + [ "$(ls | wc -l)" = $N ] || { echo "::error::expected $N files"; exit 1; } # (a test build: only the platforms its fix targets ship; the others were built as a compile check) for f in *; do case $f in *-macOS.dmg) p=macos ;; *-windows.zip) p=windows ;; *-linux-x86_64.AppImage) p=linux ;; *-linux-aarch64.AppImage) p=linux-aarch64 ;; *-iOS.laskit) p=ios ;; *-android.apk) p=android ;; + *-windows-arm64.zip) p=windows-arm64 ;; *) echo "::error::unknown file $f"; exit 1 ;; esac [[ " $PLATFORMS " == *" $p "* ]] || { echo "built, not shipped: $f"; rm "$f"; } done @@ -514,7 +592,7 @@ jobs: # (a release published on GitHub whose build failed: the "being built" line says so instead) failed: - needs: [check, macos, windows, linux, android, ios, release] + needs: [check, macos, windows, windows-arm64, linux, android, ios, release] if: failure() && github.event_name == 'release' runs-on: ubuntu-24.04 permissions: diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index d99127ca1..8176b72d5 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -19,7 +19,7 @@ android { manifestPlaceholders["appLabel"] = System.getenv("LARECOMP_APP_LABEL") ?: "LAS Recomp" minSdk = 30 targetSdk = 36 - // (experimental builds: .github/workflows/experimental.yml sets the build's version and a code that grows) + // (release.yml and nightly.yml set the build's version and a code that grows) versionCode = System.getenv("LARECOMP_VERSION_CODE")?.toInt() ?: 1 versionName = System.getenv("LARECOMP_VERSION_NAME") ?: "0.1-dev" ndk { @@ -27,7 +27,7 @@ android { } } - // The published APKs' key (.github/workflows/experimental.yml, from the repository's secrets): Android installs an + // The published APKs' key (.github/workflows/release.yml and nightly.yml, from the repository's secrets): Android installs an // update only over an app signed with the same key. Without it, the release build is unsigned. signingConfigs { System.getenv("LARECOMP_KEYSTORE")?.let { ks -> diff --git a/android/app/src/main/java/com/lasrecomp/LASRecompActivity.java b/android/app/src/main/java/com/lasrecomp/LASRecompActivity.java index e6372bfa7..2413a6afe 100644 --- a/android/app/src/main/java/com/lasrecomp/LASRecompActivity.java +++ b/android/app/src/main/java/com/lasrecomp/LASRecompActivity.java @@ -538,6 +538,10 @@ private List children(Uri tree, String id) { return out; } + // A Mac's hidden files beside the game's: ._ (a file's metadata on a drive that isn't the Mac's own, such as an + // SD card) and .DS_Store. Not the game's, so not copied: the seed generator read ._Lv07EagleTower.bntx as a dungeon. + private static boolean macJunk(String name) { return name.startsWith("._") || name.equals(".DS_Store"); } + private static Doc child(List kids, String name) { for (Doc d : kids) if (d.name.equals(name)) return d; return null; @@ -554,6 +558,7 @@ private void findGame(Uri tree, Doc dir, int depth, Doc[] found) { if (found[1] == null && rc != null && rc.isDir()) found[1] = dir; for (Doc k : kids) { String n = k.name.toLowerCase(); + if (macJunk(k.name)) continue; if (n.endsWith(".xci") || n.endsWith(".nsp") || n.endsWith(".nca") || n.endsWith(".nsz") || n.endsWith(".xcz")) unusable = true; } @@ -563,7 +568,7 @@ private void findGame(Uri tree, Doc dir, int depth, Doc[] found) { private long treeBytes(Uri tree, Doc dir) { long n = 0; - for (Doc k : children(tree, dir.id)) n += k.isDir() ? treeBytes(tree, k) : k.size; + for (Doc k : children(tree, dir.id)) n += macJunk(k.name) ? 0 : k.isDir() ? treeBytes(tree, k) : k.size; return n; } @@ -574,6 +579,7 @@ private boolean copyTree(Uri tree, Doc dir, File to) throws IOException { byte[] buf = new byte[1 << 20]; for (Doc k : children(tree, dir.id)) { if (nativeImportCancelled()) return false; + if (macJunk(k.name)) continue; File dst = new File(to, k.name); if (k.isDir()) { if (!copyTree(tree, k, dst)) return false; diff --git a/docs/BUILD-windows.md b/docs/BUILD-windows.md index 45247ab3e..1d019ffbe 100644 --- a/docs/BUILD-windows.md +++ b/docs/BUILD-windows.md @@ -62,7 +62,8 @@ above, except that the helpers need that Python passed in (there is no `py` laun Everything in the zip is ARM64 except the two helpers (`lasgen`, `rando-gen`), which are x64. AArch64 FP/SIMD instructions stay inline assembly (`--simd asm`), as on the Mac. The ARM64 zip is not part of the normal releases. -It is built by `.github/workflows/experimental.yml` for the Experimental builds page. +It is built by `.github/workflows/nightly.yml` for the nightly page, and by `release.yml` test builds that name +`windows-arm64`. Running it needs a Vulkan 1.3 driver (on Snapdragon X, Qualcomm's). To test without one, Mesa's lavapipe (a CPU Vulkan driver) for ARM64 works: set `VK_DRIVER_FILES=\lvp_icd.aarch64.json`. The Vulkan loader ignores diff --git a/docs/linux.md b/docs/linux.md index c43d9116f..bf2c81edc 100644 --- a/docs/linux.md +++ b/docs/linux.md @@ -2,9 +2,9 @@ Status: builds and runs the game on Vulkan (with your own game files) on x86-64, checked before each release by `make smoketest`. aarch64 is **experimental**: tested on a few ARM boards; reports welcome (see the end). `make appimage` / `make appimages` -package the AppImages. The latest aarch64 one is on the rolling -[Experimental builds](https://github.com/peiikai/LASRecomp/releases/tag/experimental) pre-release -(`.github/workflows/experimental.yml`); releases from 1.0.0 on carry only the x86-64 one. +package the AppImages. The latest aarch64 one is also on the rolling +[nightly](https://github.com/peiikai/LASRecomp/releases/tag/nightly) pre-release +(`.github/workflows/nightly.yml`); releases from 1.0.0 on carry only the x86-64 one. ## Running the AppImage chmod +x LAS.Recomp--linux-x86_64.AppImage diff --git a/docs/vulkan-gpu-defaults.md b/docs/vulkan-gpu-defaults.md index 997a86260..97a027bb3 100644 --- a/docs/vulkan-gpu-defaults.md +++ b/docs/vulkan-gpu-defaults.md @@ -14,6 +14,7 @@ Vendor IDs: 0x13B5 Arm (Mali), 0x5143 Qualcomm (Adreno), 0x8086 Intel, 0x1002 AM |---|---|---|---|---| | Structured shaders (static branch stacks, no jump loop) | Mali, Adreno | PanVK (RK3588), Turnip | #562 | `LARECOMP_VK_STRUCTURE` | | c1 inlined, hot constant buffer words as push constants | Mali, Adreno | PanVK; Turnip (Adreno 643) | #563, #579 | `LARECOMP_VK_PUSH_CB`, `LARECOMP_VK_INLINE_C1` | +| No constant-buffer push constants (c1 still inlined) | Qualcomm's own driver | Adreno X2-90, Windows driver E17.52.12.00: most of the picture black with them, or with fragment words only (`LARECOMP_VK_PUSH_VS=0`) the file select's text missing | #779 | `LARECOMP_VK_PUSH_CB` | | Fragment quad shuffles through derivatives (no subgroup operations in fragment shaders) | drivers that list subgroups for compute only (Turnip on Adreno 6xx) | Turnip 26.0 / 26.2, Adreno 650 (Retroid Pocket 5): a GPU hang in Marin and Tarin's house without it | | `LARECOMP_VK_QUAD_DERIV` | | One submission per command list (no 8-pass chunks) | Adreno | Turnip 26.0, Adreno 650: a GPU hang at 720p in the opening's bloom passes with chunks; house at 720p 19.5 -> 17.0 ms | | `LARECOMP_CHUNK_PASSES` | | HSV recolour in pipeline variants (a specialization constant; draws with a Cosmetics colour use the variant) | Adreno, Mali | Turnip 26.2, Adreno 650: house lit pass at 720p 8.3 -> 5.1 ms with default Cosmetics | | `LARECOMP_VK_TINT_BRANCH` | diff --git a/runtime/gamebuild.c b/runtime/gamebuild.c index 0e80865d4..189165cd5 100644 --- a/runtime/gamebuild.c +++ b/runtime/gamebuild.c @@ -75,14 +75,16 @@ static PlatProc spawn_proc(char **argv, const char *log, bool generator) { PlatProc p = 0; if (!g_quitting) { #ifdef _WIN32 - // The generator runs with Control Flow Guard off even when Windows forces it on for every program (#337) + // The generator runs with Control Flow Guard and shadow stacks off even when Windows forces them on for every + // program (#337; tools/win_longjmp.py, #779) if (generator) { - unsigned long error = 0; - p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error); + unsigned long error = 0, ss_error = 0; + p = plat_spawn_no_cfg((const char *const *)argv, log, false, &error, &ss_error); static bool told; if (p && !told) { told = true; - if (!error) LOG("[gamebuild] generator started with Control Flow Guard off\n"); + if (!error && !ss_error) LOG("[gamebuild] generator started with Control Flow Guard and shadow stacks off\n"); + else if (!error) LOG("[gamebuild] generator started with Control Flow Guard off: shadow stacks could not be turned off for it (error %lu)\n", ss_error); else LOG("[gamebuild] generator started normally: Control Flow Guard could not be turned off for it (error %lu)\n", error); } } else diff --git a/runtime/nvn/maxwell/shader.h b/runtime/nvn/maxwell/shader.h index 6a53544ac..2e8663086 100644 --- a/runtime/nvn/maxwell/shader.h +++ b/runtime/nvn/maxwell/shader.h @@ -131,6 +131,9 @@ extern bool maxwell_gmem_fs_miss; extern bool maxwell_glsl_stack_vars; extern bool maxwell_glsl_structured; // (the backend's default for this GPU: LARECOMP_VK_STATIC_STACKS / _STRUCTURE override) extern bool maxwell_glsl_push; // (the backend's default: c1 inlined + push constants; LARECOMP_VK_INLINE_C1 / _PUSH_CB override) +// The push constants part of it: the backend's default (exec.c: not on Qualcomm's own driver); LARECOMP_VK_PUSH_CB +// overrides. +extern bool maxwell_glsl_push_cb; // (GLSL) fmin / fmax / saturate as the GPU's own min / max / clamp instead of the helpers that test for NaN: for GPUs // whose min / max return the operand that isn't NaN and whose clamp to [0, 1] saturates NaN to 0 (Adreno, as D3D // requires). The backend's default (exec.c: Android); LARECOMP_VK_FAST_MINMAX=1 / =0 overrides. diff --git a/runtime/nvn/maxwell/translate.c b/runtime/nvn/maxwell/translate.c index a0181563e..47c96603b 100644 --- a/runtime/nvn/maxwell/translate.c +++ b/runtime/nvn/maxwell/translate.c @@ -30,6 +30,7 @@ bool maxwell_glsl_stack_vars = true; bool maxwell_glsl_structured; // static stacks + structured control flow by default (exec.c: Mali, Adreno) bool maxwell_glsl_fast_minmax; // the GPU's min / max / clamp for fmin / fmax / saturate (exec.c: Android) bool maxwell_glsl_push; // c1 inlined + push constants by default (exec.c: Mali, Adreno) +bool maxwell_glsl_push_cb = true; // the push constants part of it (exec.c: not on Qualcomm's own driver) bool maxwell_glsl_tint_branch; // the recolour code behind a specialization constant (exec.c: Adreno, Mali) bool maxwell_glsl_quad_deriv; // fragment quad xor shuffles through derivatives (exec.c: no subgroups in fragment shaders) const uint8_t *maxwell_const_data; @@ -2106,12 +2107,13 @@ static bool glsl_early_tests(const Ctx *c) { uint32_t maxwell_push_base, maxwell_push_cap = 64; static void push_promote(Ctx *c, Str *body) { static int on = -1; - if (on < 0) { const char *e = getenv("LARECOMP_VK_PUSH_CB"); on = e ? atoi(e) != 0 : maxwell_glsl_push; } + if (on < 0) { const char *e = getenv("LARECOMP_VK_PUSH_CB"); on = e ? atoi(e) != 0 : maxwell_glsl_push && maxwell_glsl_push_cb; } if (!on || (c->stage != STAGE_FRAGMENT && c->stage != STAGE_VERTEX) || !body->buf || (maxwell_gmem_lazy && c->info->uses_gmem)) return; - // LARECOMP_VK_PUSH_VS=0: none in vertex shaders (an A/B of the vertex words: the fragment budget stays 48 words) + // LARECOMP_VK_PUSH_VS=0: none in vertex shaders, the fragment shader's then start at offset 0 (the fragment budget + // stays 48 words). For narrowing down a driver's push constant bug (#779). static int vs_on = -1; - if (vs_on < 0) { const char *e = getenv("LARECOMP_VK_PUSH_VS"); vs_on = !e || atoi(e); } + if (vs_on < 0) { const char *e = getenv("LARECOMP_VK_PUSH_VS"); vs_on = e ? atoi(e) != 0 : 1; } if (c->stage == STAGE_VERTEX && !vs_on) return; int vs_cap = (int)maxwell_push_cap < 32 ? (int)maxwell_push_cap / 2 : 16; // (the vertex shader's share of the budget) int cap = c->stage == STAGE_VERTEX ? vs_cap : (int)maxwell_push_cap - (vs_on ? (int)maxwell_push_base / 4 : vs_cap); diff --git a/runtime/nvn/output_scale.c b/runtime/nvn/output_scale.c index af7fc9973..e6ba584bc 100644 --- a/runtime/nvn/output_scale.c +++ b/runtime/nvn/output_scale.c @@ -1158,6 +1158,39 @@ void output_scale_log(unsigned long long frame, double dw, double dh, const OutS psx = o->sx, psy = o->sy, pdw = dw, pdh = dh; } +// The render height for a mode, in units of the Switch's 1080 lines (Match display for Dynamic: its own steps come on top) +static double mode_height(int mode, double dw, double dh, bool reshape, double fit) { + if (mode >= 1 && mode <= 8) return mode == 1 && g_gpu_integrated ? 720 / 1080.0 : mode; + double h = reshape ? dh / 1080.0 : fit; // Match display + if (mode == RES_Q75) h *= 0.75; + else if (mode == RES_Q25) h *= 0.25; + else if (mode == RES_Q50) h *= 0.5; + return h; +} + +// The allocation scale for render height h and shape, capped to the GPU's texture limit +static void alloc_scale(double h, double shape, double *ax, double *ay) { + h = fmax(h, 0.1); // (.25x of a small window) + *ax = h * shape, *ay = h; + double cap = fmin(MAX_TEX / (1920.0 * *ax), MAX_TEX / (1080.0 * *ay)); + if (cap < 1.0) { *ax *= cap; *ay *= cap; } +} + +// The render scale the settings give a dw x dh window, worked out without a frame: the start screen's Video tab, which +// shows the size before the game draws anything (renderer_get_scale is 1x until the first frame). Dynamic: its top. +void output_scale_estimate(double dw, double dh, bool reshape_allowed, double *sx, double *sy) { + if (!(dw > 0) || !(dh > 0)) { *sx = *sy = 1; return; } + double fit = fmin(dw / 1920.0, dh / 1080.0), shape = (dw / dh) / (16.0 / 9.0); + const char *e = getenv("LARECOMP_WIDE_TEST"); + if (e && atof(e) > 0) shape = atof(e); + bool reshape = reshape_allowed && fabs(shape - 1.0) > 0.005; + if (!reshape) shape = 1.0; + int mode = g_settings.dyn_res ? RES_DYNAMIC : g_settings.res_mode; + double h = getenv("LARECOMP_SCALE") && atof(getenv("LARECOMP_SCALE")) > 0 ? atof(getenv("LARECOMP_SCALE")) + : mode_height(mode, dw, dh, reshape, fit); + alloc_scale(h, shape, sx, sy); +} + void output_scale(double dw, double dh, bool reshape_allowed, double now, double game_ms, double win_ms, OutScale *o) { double fit = fmin(dw / 1920.0, dh / 1080.0); static double forced = -1, wide_test = -1; @@ -1182,9 +1215,8 @@ void output_scale(double dw, double dh, bool reshape_allowed, double now, double } p_mode = mode, p_ups = g_settings.upscaler, p_aa = g_settings.antialias, p_fps = settings_fps_target(); if (forced > 0) h = forced; - else if (mode >= 1 && mode <= 8) h = mode == 1 && g_gpu_integrated ? 720 / 1080.0 : mode; else { - h = reshape ? dh / 1080.0 : fit; // Match display + h = mode_height(mode, dw, dh, reshape, fit); if (mode == RES_DYNAMIC) { static double pinned = -1; if (pinned < 0) pinned = getenv("LARECOMP_DYN_TEST") ? atof(getenv("LARECOMP_DYN_TEST")) : 0; @@ -1192,14 +1224,9 @@ void output_scale(double dw, double dh, bool reshape_allowed, double now, double else { dyn_update(now, game_ms, win_ms, h * 1080); dyn = g_dyn; } g_dyn_active = pinned <= 0; } - else if (mode == RES_Q75) h *= 0.75; - else if (mode == RES_Q25) h *= 0.25; - else if (mode == RES_Q50) h *= 0.5; } - h = fmax(h, 0.1); // (.25x of a small window) - double ax = h * shape, ay = h; - double cap = fmin(MAX_TEX / (1920.0 * ax), MAX_TEX / (1080.0 * ay)); - if (cap < 1.0) { ax *= cap; ay *= cap; } + double ax, ay; + alloc_scale(h, shape, &ax, &ay); o->ax = ax; o->ay = ay; o->sx = ax * dyn; o->sy = ay * dyn; o->shape = shape; g_render_w = (int)lround(1920.0 * o->sx), g_render_h = (int)lround(1080.0 * o->sy); diff --git a/runtime/nvn/output_scale.h b/runtime/nvn/output_scale.h index 56b78c3f9..75e898819 100644 --- a/runtime/nvn/output_scale.h +++ b/runtime/nvn/output_scale.h @@ -14,6 +14,8 @@ typedef struct { // unknown), win_ms the window step's - the game's image drawn into the window (upscale, SMAA) and the menu (< 0: // unknown / not counted) void output_scale(double dw, double dh, bool reshape_allowed, double now, double game_ms, double win_ms, OutScale *o); +// The render scale the settings give a dw x dh window, without a frame (the start screen; Dynamic: its top) +void output_scale_estimate(double dw, double dh, bool reshape_allowed, double *sx, double *sy); double output_scale_dynamic_factor(void); // Dynamic resolution for the frame pacer (pace.c "Resolution first"): 0 off (a fixed resolution, or pinned for a test), // 1 on with nothing lower to try (at its floor, the Minimum resolution; or holding a frame rate itself: 60 out of reach), diff --git a/runtime/nvn/vulkan/device.c b/runtime/nvn/vulkan/device.c index d893c3bfe..fe0305302 100644 --- a/runtime/nvn/vulkan/device.c +++ b/runtime/nvn/vulkan/device.c @@ -724,6 +724,7 @@ static void create_device(void) { if (pbin_ext) pbp.pNext = pr.pNext, pr.pNext = &pbp; #endif vkGetPhysicalDeviceProperties2(g_vk.phys, &pr); + g_vk.driver_id = drv.driverID; // the heap as driver memory (exec.c guest_heap_dmabuf) by default: RADV, and PanVK (Mali-G610, Mesa 27.0-devel, // issue #483: an 81-minute run without faults, the Field 15 -> 21 fps, ~85% fewer uploads). PanVK's device name // is the GPU's ("Mali-G610 MC4"), hence the driver ID. diff --git a/runtime/nvn/vulkan/exec.c b/runtime/nvn/vulkan/exec.c index 68ae0efe0..023e00958 100644 --- a/runtime/nvn/vulkan/exec.c +++ b/runtime/nvn/vulkan/exec.c @@ -1382,10 +1382,14 @@ void vk_exec_init(void) { // constant buffers from memory every time. Turnip loads them into its constant file in a per-draw preamble, which // push constants shorten (the field's main lit pass on an Adreno 643: 7.52 -> 6.66 ms, preamble 386 -> 304 // instructions). LARECOMP_VK_INLINE_C1 / LARECOMP_VK_PUSH_CB =1 / =0 override. + // The push constants, not on Qualcomm's own driver (#779: Adreno X2-90, Windows driver E17.52.12.00). With words in + // both stages most of the picture was black; with fragment words alone (LARECOMP_VK_PUSH_VS=0) the world drew but + // the file select's text didn't. Without them everything drew. c1 inlined stays on: those runs had it. maxwell_glsl_push = g_vk.props.vendorID == 0x13B5 || g_vk.props.vendorID == 0x5143; // (Arm, Qualcomm) + maxwell_glsl_push_cb = g_vk.driver_id != VK_DRIVER_ID_QUALCOMM_PROPRIETARY; const char *pe = getenv("LARECOMP_VK_PUSH_CB"); - LOG("[vk] constant-buffer push constants %s (%s)\n", (pe ? atoi(pe) != 0 : maxwell_glsl_push) ? "on" : "off", - pe ? "LARECOMP_VK_PUSH_CB" : "default for this GPU"); + LOG("[vk] constant-buffer push constants %s (%s)\n", (pe ? atoi(pe) != 0 : maxwell_glsl_push && maxwell_glsl_push_cb) ? "on" : "off", + pe ? "LARECOMP_VK_PUSH_CB" : maxwell_glsl_push_cb ? "default for this GPU" : "Qualcomm's driver"); { // fragment quad xor shuffles through derivatives (shader.h maxwell_glsl_quad_deriv) where the driver has no subgroup // operations in fragment shaders: Turnip on Adreno 6xx lists compute only, and the shaders' subgroupShuffleXor // hung the GPU in Marin and Tarin's house (the lit pass behind the depth prepass; Retroid Pocket 5, Adreno 650). diff --git a/runtime/nvn/vulkan/vk.h b/runtime/nvn/vulkan/vk.h index 3f3f6629f..e8640bc45 100644 --- a/runtime/nvn/vulkan/vk.h +++ b/runtime/nvn/vulkan/vk.h @@ -46,6 +46,7 @@ typedef struct { uint32_t ts_bits; // the queue's timestampValidBits (0: no timestamps) VmaAllocator vma; VkPhysicalDeviceProperties props; + VkDriverId driver_id; // (device.c; 0 when the driver doesn't say) bool cpu_astc; // ASTC decoded on the CPU although the driver lists it (device.c) VkPipelineCache pcache; VkSemaphore timeline; // value = last completed submission number diff --git a/runtime/platform.h b/runtime/platform.h index b1e054895..090cc2dca 100644 --- a/runtime/platform.h +++ b/runtime/platform.h @@ -160,9 +160,12 @@ int plat_proc_wait(PlatProc p, bool block); // exit code, or -1 while still run #define PLAT_PROC_SIGNALED 256 // + N: ended by signal N instead of exiting (not on Windows) #define PLAT_PROC_FAST_FAIL 512 // Windows: ended by a security check (0xC0000409, e.g. Control Flow Guard) #ifdef _WIN32 -// plat_spawn with Control Flow Guard off for the child (the generator: unicorn's longjmp fails CFG's check, #337). -// *error: 0 = started that way; else why not (then it was started normally, or not at all if that failed too) -PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error); +// plat_spawn with Control Flow Guard and shadow stacks off for the child (the generator: unicorn's longjmp fails +// CFG's check, #337, and can't unwind with a shadow stack; tools/win_longjmp.py, #779). +// *error: 0 = started with CFG off; else why not (then it was started normally, or not at all if that failed too). +// *shadow_stack_error: the same for shadow stacks (when only that is refused, it was started with CFG off alone). +PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error, + unsigned long *shadow_stack_error); #endif void plat_proc_kill(PlatProc p); // end it and what it started (then it's gone: no wait) bool plat_remove_tree(const char *dir); // rm -rf diff --git a/runtime/platform/windows.c b/runtime/platform/windows.c index 24a454e15..3d64d0b30 100644 --- a/runtime/platform/windows.c +++ b/runtime/platform/windows.c @@ -767,18 +767,28 @@ ssize_t pwrite(int fd, const void *buf, size_t n, off_t off) { return prw(fd, (v // ---- child processes typedef struct { HANDLE process, job; } WinProcess; -// Control Flow Guard off for one child (plat_spawn_no_cfg), as a process mitigation policy: it wins over the -// system-wide Exploit protection setting, which the exe's header (tools/pe_no_cfg.py) does not (#337) +// Control Flow Guard and user-mode shadow stacks off for one child (plat_spawn_no_cfg), as process mitigation +// policies: they win over the system-wide Exploit protection settings, which the exe's header (tools/pe_no_cfg.py) +// does not. unicorn (the generator's jump tables) leaves the code it generated with longjmp from a setjmp that saved +// no frame, and VCRUNTIME140's longjmp stops the process (0xC0000409) for that when CFG is on (#337); with a shadow +// stack it would unwind instead (RtlUnwindEx), through generated code that has no unwind data. Under x64 emulation +// on ARM (Windows 11 26H1) the generator still ended there with both off, so it now replaces unicorn's setjmp and +// longjmp with its own (tools/win_longjmp.py, #779). #ifndef PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY #define PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY 0x00020007 #endif #ifndef PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF #define PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF (0x00000002ull << 40) #endif +#ifndef PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF +#define PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF (0x00000002ull << 28) +#endif // no_cfg: NULL = a plain start; else *no_cfg = 0 when the child runs with CFG off, or the error that prevented it -// (then it is started without the policy) -static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg) { +// (then it is started without the policy). *no_ss likewise for shadow stacks: a Windows that doesn't take that +// policy gets CFG off alone. +static PlatProc spawn_child(const char *const argv[], const char *log_path, bool append_log, unsigned long *no_cfg, + unsigned long *no_ss) { static __thread char cmd[32768]; cmd[0] = 0; for (int i = 0; argv[i]; i++) append_arg(cmd, sizeof cmd, argv[i]); @@ -800,29 +810,36 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool const wchar_t *app = GetFullPathNameW(wprog, MAX_PATH, wfull, NULL) ? wfull : NULL; BOOL ok = FALSE; if (no_cfg) { - *no_cfg = ERROR_NOT_SUPPORTED; - STARTUPINFOEXW six = {.StartupInfo = si}; - six.StartupInfo.cb = sizeof six; - DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, 0}; - SIZE_T size = 0; - InitializeProcThreadAttributeList(NULL, 1, 0, &size); - six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL; - if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) { - *no_cfg = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY; - free(six.lpAttributeList); - six.lpAttributeList = NULL; - } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, policy, - sizeof policy, NULL, NULL)) { - *no_cfg = GetLastError(); - } else { - ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, - CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL, - &six.StartupInfo, &pi); - *no_cfg = ok ? 0 : GetLastError(); // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it) - } - if (six.lpAttributeList) { - DeleteProcThreadAttributeList(six.lpAttributeList); - free(six.lpAttributeList); + *no_cfg = *no_ss = ERROR_NOT_SUPPORTED; + // first both policies, then CFG alone (a Windows without shadow stack support may refuse the second word) + for (int both = 1; both >= 0 && !ok; both--) { + STARTUPINFOEXW six = {.StartupInfo = si}; + six.StartupInfo.cb = sizeof six; + DWORD64 policy[2] = {PROCESS_CREATION_MITIGATION_POLICY_CONTROL_FLOW_GUARD_ALWAYS_OFF, + both ? PROCESS_CREATION_MITIGATION_POLICY2_CET_USER_SHADOW_STACKS_ALWAYS_OFF : 0}; + unsigned long e; + SIZE_T size = 0; + InitializeProcThreadAttributeList(NULL, 1, 0, &size); + six.lpAttributeList = size ? (LPPROC_THREAD_ATTRIBUTE_LIST)malloc(size) : NULL; + if (!six.lpAttributeList || !InitializeProcThreadAttributeList(six.lpAttributeList, 1, 0, &size)) { + e = six.lpAttributeList ? GetLastError() : ERROR_NOT_ENOUGH_MEMORY; + free(six.lpAttributeList); + six.lpAttributeList = NULL; + } else if (!UpdateProcThreadAttribute(six.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_MITIGATION_POLICY, + policy, sizeof policy, NULL, NULL)) { + e = GetLastError(); + } else { + ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, + CREATE_NO_WINDOW | CREATE_SUSPENDED | EXTENDED_STARTUPINFO_PRESENT, NULL, NULL, + &six.StartupInfo, &pi); + e = ok ? 0 : GetLastError(); // (older Windows: ERROR_INVALID_PARAMETER; or a policy refuses it) + } + if (six.lpAttributeList) { + DeleteProcThreadAttributeList(six.lpAttributeList); + free(six.lpAttributeList); + } + if (both) *no_ss = e; + if (ok || !both) *no_cfg = e; } } if (!ok) ok = CreateProcessW(app, wcmd, NULL, NULL, TRUE, CREATE_NO_WINDOW | CREATE_SUSPENDED, NULL, NULL, &si, &pi); @@ -855,13 +872,15 @@ static PlatProc spawn_child(const char *const argv[], const char *log_path, bool } PlatProc plat_spawn(const char *const argv[], const char *log_path, bool append_log) { - return spawn_child(argv, log_path, append_log, NULL); + return spawn_child(argv, log_path, append_log, NULL, NULL); } -PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error) { - unsigned long e = 0; - PlatProc p = spawn_child(argv, log_path, append_log, &e); +PlatProc plat_spawn_no_cfg(const char *const argv[], const char *log_path, bool append_log, unsigned long *error, + unsigned long *shadow_stack_error) { + unsigned long e = 0, ss = 0; + PlatProc p = spawn_child(argv, log_path, append_log, &e, &ss); if (error) *error = e; + if (shadow_stack_error) *shadow_stack_error = ss; return p; } diff --git a/runtime/ui/launcher.mm b/runtime/ui/launcher.mm index 52655c230..5dc39e466 100644 --- a/runtime/ui/launcher.mm +++ b/runtime/ui/launcher.mm @@ -119,7 +119,7 @@ static void save_png(id tex, const char *path) { // players move to it. #define LARECOMP_VERSION "v0.6.0" #define LARECOMP_TEST_OF "v0.6.0" -// Experimental builds (.github/workflows/experimental.yml sets these): the build's time (YYYYMMDDHHMM, UTC; 0: not an +// Experimental builds (the retired experimental.yml set these; Experimental moved to nightly): the build's time (YYYYMMDDHHMM, UTC; 0: not an // experimental build). They only look for a newer experimental build of their own platform, never a normal release. #define LARECOMP_EXPERIMENTAL_STAMP 0LL // Nightly builds (.github/workflows/nightly.yml sets this): the build's time (YYYYMMDDHHMM, UTC; 0: not a nightly). diff --git a/runtime/ui/menu.mm b/runtime/ui/menu.mm index f23283f3a..b1bbf98bc 100644 --- a/runtime/ui/menu.mm +++ b/runtime/ui/menu.mm @@ -651,6 +651,10 @@ static void drain_events(ImGuiIO &io) { } extern "C" void renderer_get_scale(float *sx, float *sy); +extern "C" bool widescreen_allowed(void); // runtime/hle/widescreen.c +// The settings pages are drawn in game (tabs_body's in_game): on the start screen no frame has been drawn yet, so the +// renderer's scale is still 1x and the Resolution row works its size out from the settings and the window instead +static bool g_tabs_in_game; extern "C" int input_pad_family(void); // runtime/hle/input.c extern "C" double output_scale_dynamic_factor(void); #ifdef __APPLE__ @@ -744,7 +748,12 @@ static void tab_display(void) { if (res_mode[i] == g_default_res_mode) rdef = i; } float sx, sy; - renderer_get_scale(&sx, &sy); + if (g_tabs_in_game) renderer_get_scale(&sx, &sy); + else { + double ex, ey; + output_scale_estimate(g_win_px_w, g_win_px_h, widescreen_allowed(), &ex, &ey); + sx = (float)ex, sy = (float)ey; + } char d[1024]; // (in the menus' language already: the row leaves it as it is) snprintf(d, sizeof d, g_gpu_integrated @@ -1455,6 +1464,7 @@ static void page_corner_icon(int tab, float scale) { // first tab, game files and fonts, drawn by launcher.mm). only: one page alone, without the tab bar (the in-game menu // bar's pages but Settings). static void tabs_body(float scale, bool in_game, void (*prelaunch)(float), int only = -1000) { + g_tabs_in_game = in_game; // (id: the page; -1 = Prelaunch) struct Tab { const char *name; int id; bool on; }; Tab tabs[12]; diff --git a/runtime/ui/update_tags.h b/runtime/ui/update_tags.h index 16a27b75c..0475eab13 100644 --- a/runtime/ui/update_tags.h +++ b/runtime/ui/update_tags.h @@ -44,7 +44,7 @@ static inline std::string tag_label(const std::string &t, bool with_word) { // Rolling builds are never on the normal releases: each kind sits on one rolling pre-release named after it, its files // named LAS.Recomp----.. Kinds: "experimental" (experimental platforms, -// .github/workflows/experimental.yml) and "nightly" (every platform from develop, .github/workflows/nightly.yml). +// from the retired experimental.yml) and "nightly" (every platform from develop, .github/workflows/nightly.yml). // The newest stamp of that kind for this platform (e.g. "linux-aarch64"), found in the releases JSON; 0: none. static inline long long newest_rolling(const std::string &body, const std::string &kind, const std::string &platform) { const std::string key = "LAS.Recomp-" + kind + "-"; diff --git a/tools/check_apk.sh b/tools/check_apk.sh index b985b8259..87786c5b1 100755 --- a/tools/check_apk.sh +++ b/tools/check_apk.sh @@ -2,7 +2,7 @@ # tools/check_apk.sh : the APK holds only what the app ships - the runtime, SDL, shaderc, FFmpeg, the shipped fonts # and icon with their licences, and the game code builder in assets/gen.zip (Python's library, the bindings, zig's # lib/, the generator and the headers, and the seed generator: tools/rando, LAS-Randomizer's code and data), checked -# file by file. Nothing of the game. (.github/workflows/ci.yml, experimental.yml) +# file by file. Nothing of the game. (.github/workflows/ci.yml, release.yml, nightly.yml) set -eu APK=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") T=$(mktemp -d); trap 'rm -rf "$T"' EXIT diff --git a/tools/gen.py b/tools/gen.py index 790aa490b..c30ddd1bb 100644 --- a/tools/gen.py +++ b/tools/gen.py @@ -23,6 +23,7 @@ import lifter from lifter import CANON, Insn, Lifter, Unsupported, bits, emit_asm_batch, gaddr, hx, sext from nso import Nso, R_AARCH64_RELATIVE +import win_longjmp # (Windows: unicorn's longjmp, #779) SCRATCH_MEM = 0x7300000000 @@ -89,6 +90,8 @@ def boundaries(self): def emu(self): if self.mu is None: + # (the first unicorn run, for the first jump table: a generator stopped by Windows ends here, #337, #779) + print(f'{self.name}: starting the jump table emulator', file=sys.stderr, flush=True) mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM) n = self.nso mu.mem_map(self.base, len(n.image)) @@ -553,6 +556,9 @@ def main(): help='Android: guest addresses relative to the window the runtime picks (lifter.gaddr)') args = ap.parse_args() lifter.DYN_BASE = args.dyn_base + # (first: a generator that ends early still shows it started, and where it ran) + where = ', '.join(x for x in (win_longjmp.describe(), win_longjmp.apply()) if x) + print(f'{args.name}: started' + (f' ({where})' if where else ''), file=sys.stderr, flush=True) t0 = time.time() m = Module(args.name, args.nso, args.base, args.simd) diff --git a/tools/rando/las_headless.py b/tools/rando/las_headless.py index dfea0b052..263f609b3 100644 --- a/tools/rando/las_headless.py +++ b/tools/rando/las_headless.py @@ -14,6 +14,21 @@ import sys import types +# Game files copied from a Mac to a drive that isn't its own (an SD card, exFAT) carry a hidden ._ beside each +# file. They aren't the game's, but every folder listing here took them for its files: the room shuffle read +# ._Lv07EagleTower.bntx as a dungeon's textures and failed. One filter for every listing (tools/rando and LAS-Randomizer). +_listdir = os.listdir + + +def _listdir_game(path='.'): + names = _listdir(path) + if names and isinstance(names[0], bytes): + return [f for f in names if not (f.startswith(b'._') or f == b'.DS_Store')] + return [f for f in names if not (f.startswith('._') or f == '.DS_Store')] + + +os.listdir = _listdir_game + if os.environ.get('LARECOMP_RANDO_SCRATCH'): # (Android: the app's internal storage - see shuffle_dungeon_rooms) import tempfile tempfile.tempdir = os.environ['LARECOMP_RANDO_SCRATCH'] diff --git a/tools/test_data_move.py b/tools/test_data_move.py index 07e35d846..6eee2f2db 100644 --- a/tools/test_data_move.py +++ b/tools/test_data_move.py @@ -17,8 +17,9 @@ deps = os.environ.get('DEPS') # (a prefix with SDL3, as for make DEPS=) if deps: flags = [f'-I{deps}/include', f'-L{deps}/lib', f'-Wl,-rpath,{deps}/lib', '-lSDL3'] -elif shutil.which('pkg-config') and subprocess.run(['pkg-config', '--exists', 'sdl3']).returncode == 0: - flags = shlex.split(subprocess.run(['pkg-config', '--cflags', '--libs', 'sdl3'], capture_output=True, text=True, +# MSYS2's pkgconf 3.0.11 installs pkg-config without .exe, which shutil.which can't find on Windows; pkgconf.exe stays +elif (pc := shutil.which('pkg-config') or shutil.which('pkgconf')) and subprocess.run([pc, '--exists', 'sdl3']).returncode == 0: + flags = shlex.split(subprocess.run([pc, '--cflags', '--libs', 'sdl3'], capture_output=True, text=True, check=True).stdout) elif sys.platform == 'darwin': brew = subprocess.run(['brew', '--prefix'], capture_output=True, text=True).stdout.strip() or '/opt/homebrew' diff --git a/tools/test_delete_guard.py b/tools/test_delete_guard.py index 893f123a6..d4051c2f1 100644 --- a/tools/test_delete_guard.py +++ b/tools/test_delete_guard.py @@ -14,8 +14,9 @@ deps = os.environ.get('DEPS') # (a prefix with SDL3, as for make DEPS=) if deps: flags = [f'-I{deps}/include', f'-L{deps}/lib', f'-Wl,-rpath,{deps}/lib', '-lSDL3'] -elif shutil.which('pkg-config') and subprocess.run(['pkg-config', '--exists', 'sdl3']).returncode == 0: - flags = shlex.split(subprocess.run(['pkg-config', '--cflags', '--libs', 'sdl3'], capture_output=True, text=True, +# MSYS2's pkgconf 3.0.11 installs pkg-config without .exe, which shutil.which can't find on Windows; pkgconf.exe stays +elif (pc := shutil.which('pkg-config') or shutil.which('pkgconf')) and subprocess.run([pc, '--exists', 'sdl3']).returncode == 0: + flags = shlex.split(subprocess.run([pc, '--cflags', '--libs', 'sdl3'], capture_output=True, text=True, check=True).stdout) elif sys.platform == 'darwin': brew = subprocess.run(['brew', '--prefix'], capture_output=True, text=True).stdout.strip() or '/opt/homebrew' diff --git a/tools/test_gamebuild_ios.py b/tools/test_gamebuild_ios.py index ac403c5a3..61a79a4af 100644 --- a/tools/test_gamebuild_ios.py +++ b/tools/test_gamebuild_ios.py @@ -54,7 +54,7 @@ extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); } extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); } -extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); } +extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); } extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); } extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); } ''' diff --git a/tools/test_gamebuild_resume.py b/tools/test_gamebuild_resume.py index b6398e257..d2c1bb7b5 100644 --- a/tools/test_gamebuild_resume.py +++ b/tools/test_gamebuild_resume.py @@ -44,7 +44,7 @@ extern "C" uint64_t plat_now_ns() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } extern "C" void plat_sleep_until_ns(uint64_t t) { auto now=plat_now_ns(); if(t>now)std::this_thread::sleep_for(std::chrono::nanoseconds(t-now)); } extern "C" intptr_t plat_spawn(const char *const *a,const char *l,bool append) { return start_cb(a,l,append); } -extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e) { if(e)*e=0; return start_cb(a,l,append); } +extern "C" intptr_t plat_spawn_no_cfg(const char *const *a,const char *l,bool append,unsigned long *e,unsigned long *ss) { if(e)*e=0; if(ss)*ss=0; return start_cb(a,l,append); } extern "C" int plat_proc_wait(intptr_t p,bool block) { return wait_cb(p,block); } extern "C" void plat_proc_kill(intptr_t p) { kill_cb(p); } ''' diff --git a/tools/test_translate.sh b/tools/test_translate.sh index f68a9a3b5..947c4b71a 100755 --- a/tools/test_translate.sh +++ b/tools/test_translate.sh @@ -23,7 +23,7 @@ RENAME="-Dmaxwell_translate=old_maxwell_translate -Dmaxwell_free=old_maxwell_fre -Dmaxwell_gmem_chunk_shift=old_maxwell_gmem_chunk_shift -Dmaxwell_gmem_lazy=old_maxwell_gmem_lazy -Dmaxwell_gmem_fs_miss=old_maxwell_gmem_fs_miss -Dmaxwell_glsl_stack_vars=old_maxwell_glsl_stack_vars -Dmaxwell_gmem_mirror=old_maxwell_gmem_mirror -Dmaxwell_glsl_structured=old_maxwell_glsl_structured - -Dmaxwell_glsl_push=old_maxwell_glsl_push -Dmaxwell_const_data=old_maxwell_const_data + -Dmaxwell_glsl_push=old_maxwell_glsl_push -Dmaxwell_glsl_push_cb=old_maxwell_glsl_push_cb -Dmaxwell_const_data=old_maxwell_const_data -Dmaxwell_const_size=old_maxwell_const_size -Dmaxwell_push_base=old_maxwell_push_base -Dmaxwell_push_cap=old_maxwell_push_cap -Dmaxwell_glsl_fast_minmax=old_maxwell_glsl_fast_minmax -Dmaxwell_glsl_quad_deriv=old_maxwell_glsl_quad_deriv -Dmaxwell_glsl_tint_branch=old_maxwell_glsl_tint_branch" diff --git a/tools/test_win_longjmp.py b/tools/test_win_longjmp.py new file mode 100644 index 000000000..0e07da657 --- /dev/null +++ b/tools/test_win_longjmp.py @@ -0,0 +1,60 @@ +"""Windows, x64 Python with unicorn (what lasgen is built from): unicorn's emulation with VCRUNTIME140's longjmp and +with tools/win_longjmp.py's (#779). The first may be stopped (0xC0000409) under x64 emulation on Windows on ARM: that is +reported, not failed. The second must work everywhere, including emulation errors (also a longjmp). +Run: python tools/test_win_longjmp.py (with `pip install unicorn==2.1.4`, as the Makefile's gen-helper).""" +import os +import subprocess +import sys + +if os.name != 'nt': + raise SystemExit('Windows-only test') + +TOOLS = os.path.dirname(os.path.abspath(__file__)) + + +def child(fixed): + sys.path.insert(0, TOOLS) + import unicorn as uc + from unicorn import arm64_const as A + import win_longjmp + print('own' if fixed else 'VCRUNTIME140', 'longjmp:', win_longjmp.describe(), flush=True) + if fixed: + r = win_longjmp.apply() + assert r == 'own setjmp/longjmp', r + base = 0x7004000000 + mu = uc.Uc(uc.UC_ARCH_ARM64, uc.UC_MODE_ARM) + mu.mem_map(base, 0x1000) + # add x0, x0, #1 ; add x1, x1, #2 ; ldr x2, [x3] ; br x4 + mu.mem_write(base, bytes.fromhex('00040091 21080091 620040f9 80001fd6')) + for i in range(2000): # (one instruction at a time, as gen.py's jump tables: every stop leaves by longjmp) + mu.reg_write(A.UC_ARM64_REG_X0, i) + mu.emu_start(base, 0, count=1) + assert mu.reg_read(A.UC_ARM64_REG_X0) == i + 1 + mu.reg_write(A.UC_ARM64_REG_X1, 5) + mu.emu_start(base, base + 8) + assert mu.reg_read(A.UC_ARM64_REG_X1) == 7 + mu.reg_write(A.UC_ARM64_REG_X3, 0x123450000) # (unmapped: the error path) + for _ in range(100): + try: + mu.emu_start(base + 8, 0, count=1) + raise AssertionError('read of unmapped memory did not fail') + except uc.UcError as e: + assert e.errno == uc.UC_ERR_READ_UNMAPPED, e + print('ok', flush=True) + + +if len(sys.argv) > 1: + child(sys.argv[1] == 'fixed') + sys.exit(0) + +for fixed in (False, True): + r = subprocess.run([sys.executable, __file__, 'fixed' if fixed else 'plain'], capture_output=True, text=True, + timeout=120) + sys.stdout.write(r.stdout + r.stderr) + code = r.returncode & 0xFFFFFFFF + if fixed: + assert code == 0 and r.stdout.rstrip().endswith('ok'), f'own longjmp: exit {code:#x}' + elif code: + print(f'::notice::unicorn with VCRUNTIME140 longjmp: exit {code:#x}' + + (' (fast fail: the #779 case)' if code == 0xC0000409 else '')) +print("unicorn's longjmp: emulation and its errors with tools/win_longjmp.py passed") diff --git a/tools/test_windows_processes.py b/tools/test_windows_processes.py index a25df7fe6..59a25e257 100644 --- a/tools/test_windows_processes.py +++ b/tools/test_windows_processes.py @@ -45,10 +45,15 @@ CloseHandle(pi.hThread);CloseHandle(pi.hProcess);Sleep(30000);return 0; } if(argc>1&&!strcmp(argv[1],"--fastfail")) { ExitProcess(0xC0000409u); } - if(argc>1&&!strcmp(argv[1],"--cfg")) { // 0: Control Flow Guard off in this process + if(argc>1&&!strcmp(argv[1],"--cfg")) { // 0: Control Flow Guard and shadow stacks off in this process PROCESS_MITIGATION_CONTROL_FLOW_GUARD_POLICY p={0}; if(!GetProcessMitigationPolicy(GetCurrentProcess(),ProcessControlFlowGuardPolicy,&p,sizeof p)) return 3; - return p.EnableControlFlowGuard?1:0; + DWORD ss=0; // (ProcessUserShadowStackPolicy = 17: bit 0 EnableUserShadowStack. x64 only: an ARM64 + // process reads it as on even when started with it always off - Windows on ARM runner) +#ifdef __x86_64__ + if(!GetProcessMitigationPolicy(GetCurrentProcess(),(PROCESS_MITIGATION_POLICY)17,&ss,sizeof ss)) return 4; +#endif + return p.EnableControlFlowGuard?1:(ss&1)?2:0; } assert(argc==2); char a[1200],b[1200];snprintf(a,sizeof a,"%s/first.pid",argv[1]);snprintf(b,sizeof b,"%s/second.pid",argv[1]); @@ -66,16 +71,19 @@ assert(plat_proc_wait(plat_spawn(crash,NULL,false),true)==255); const char *fastfail[]={exe,"--fastfail",NULL},*cfg[]={exe,"--cfg",NULL}; assert(plat_proc_wait(plat_spawn(fastfail,NULL,false),true)==PLAT_PROC_FAST_FAIL); - unsigned long cfg_error=1; - PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error); - assert(nocfg&&cfg_error==0&&plat_proc_wait(nocfg,true)==0); + unsigned long cfg_error=1,ss_error=1; + PlatProc nocfg=plat_spawn_no_cfg(cfg,NULL,false,&cfg_error,&ss_error); + int nocfg_exit=nocfg?plat_proc_wait(nocfg,true):-2; + if(!nocfg||cfg_error||ss_error||nocfg_exit) + fprintf(stderr,"CFG-off start: cfg error %lu, shadow stack error %lu, child %d\n",cfg_error,ss_error,nocfg_exit); + assert(nocfg&&cfg_error==0&&ss_error==0&&nocfg_exit==0); reject_job=true; assert(plat_proc_wait(plat_spawn(code,NULL,false),true)==7); const char *sleeper[]={exe,"--sleep",NULL}; PlatProc fallback=plat_spawn(sleeper,NULL,false); assert(fallback && !((WinProcess *)fallback)->job && plat_proc_wait(fallback,false)==-1); plat_proc_kill(fallback); - puts("Windows process trees: isolated cancellation, crash status, CFG-off start and fallback without job assignment passed"); + puts("Windows process trees: isolated cancellation, crash status, CFG- and shadow-stack-off start and fallback without job assignment passed"); return 0; } ''' diff --git a/tools/win_longjmp.py b/tools/win_longjmp.py new file mode 100644 index 000000000..81d29d86a --- /dev/null +++ b/tools/win_longjmp.py @@ -0,0 +1,176 @@ +"""Windows (x64 lasgen, also under x64 emulation on ARM): unicorn's setjmp / longjmp without VCRUNTIME140's (#337, #779). + +unicorn leaves the code it generated with longjmp, from a setjmp its wrapper calls with frame 0 so that longjmp +restores the registers instead of unwinding through generated code, which has no unwind data. VCRUNTIME140's longjmp +doesn't always do that: +- when its Control Flow Guard check pointer isn't its own no-op, it fast-fails (0xC0000409) on frame 0. On x64 + Windows that is when CFG is on (#337: the generator is started with CFG off). Under x64 emulation on ARM, Windows + may set that pointer with CFG off as well: it is how calls into ARM64EC code are caught (Microsoft's ARM64EC notes). +- when there is a shadow stack (rdsspq gives one), it always unwinds with RtlUnwindEx, which fails in generated code. +On Windows 11 26H1 on ARM every first jump table emulation ended the generator there (#779), even started with CFG +and shadow stacks off; the Windows on ARM CI runner (an older Windows) runs VCRUNTIME140's fine. + +apply() points unicorn.dll's setjmp and longjmp imports at SETJMP_X64 / LONGJMP_X64: a plain register save and +restore (what VCRUNTIME140's longjmp does with frame 0 and no checks) that also pops a shadow stack, when there is +one, back to where setjmp was (setjmp keeps it in the frame field, which only LONGJMP_X64 reads).""" +import ctypes +import os +import struct + +# x64, rcx = the jmp_buf (_JUMP_BUFFER: Frame, Rbx, Rsp, Rbp, Rsi, Rdi, R12-R15, Rip, MxCsr, FpCsr, Xmm6-15), rdx unused: +# xor eax, eax / rdsspq rax / test rax, rax / jz 1f / add rax, 8 (the shadow stack after this ret; 0: none) +# 1: mov [rcx], rax / mov [rcx+8], rbx / lea r8, [rsp+8] / mov [rcx+0x10], r8 / mov [rcx+0x18..0x48], rbp rsi rdi +# r12-r15 / mov r8, [rsp] / mov [rcx+0x50], r8 / stmxcsr [rcx+0x58] / fnstcw [rcx+0x5c] +# movdqu [rcx+0x60..0xf0], xmm6-xmm15 / xor eax, eax / ret +SETJMP_X64 = bytes.fromhex( + '31c0f3480f1ec84885c074044883c008488901488959084c8d4424084c8941104889691848897120488979284c8961304c8969384c897140' + '4c8979484c8b04244c8941500fae5958d9795cf30f7f7160f30f7f7970f3440f7f8180000000f3440f7f8990000000f3440f7f91a0000000' + 'f3440f7f99b0000000f3440f7fa1c0000000f3440f7fa9d0000000f3440f7fb1e0000000f3440f7fb9f000000031c0c3') +# rcx = the jmp_buf, edx = the value: +# mov r8, [rcx] / test r8, r8 / jz 3f / xor r9d, r9d / rdsspq r9 / test r9, r9 / jz 3f +# sub r8, r9 / shr r8, 3 / 2: cmp r8, 255 / jbe 4f / mov r9d, 255 / incsspq r9 / sub r8, 255 / jmp 2b / 4: incsspq r8 +# 3: mov eax, edx / test eax, eax / jnz 5f / inc eax (longjmp(b, 0) returns 1) +# 5: mov rbx, [rcx+8] / mov rsp, [rcx+0x10] / mov rbp..r15, [rcx+0x18..0x48] / ldmxcsr [rcx+0x58] / fnclex +# fldcw [rcx+0x5c] / movdqu xmm6-xmm15, [rcx+0x60..0xf0] / jmp qword ptr [rcx+0x50] +LONGJMP_X64 = bytes.fromhex( + '4c8b014d85c074364531c9f3490f1ec94d85c974294d29c849c1e8034981f8ff000000761441b9ff000000f3490faee94981e8ff000000' + 'ebe3f3490faee889d085c07502ffc0488b5908488b6110488b6918488b7120488b79284c8b61304c8b69384c8b71404c8b79480fae5158' + 'dbe2d9695cf30f6f7160f30f6f7970f3440f6f8180000000f3440f6f8990000000f3440f6f91a0000000f3440f6f99b0000000f3440f6f' + 'a1c0000000f3440f6fa9d0000000f3440f6fb1e0000000f3440f6fb9f0000000ff6150') + +_stubs = None # (kept for the life of the process) + + +def _k32(): + from ctypes import wintypes as w + k = ctypes.WinDLL('kernel32', use_last_error=True) + k.GetCurrentProcess.restype = w.HANDLE + k.GetCurrentProcess.argtypes = [] + k.GetModuleHandleW.restype = w.HMODULE + k.GetModuleHandleW.argtypes = [w.LPCWSTR] + k.VirtualAlloc.restype = ctypes.c_void_p + k.VirtualAlloc.argtypes = [ctypes.c_void_p, ctypes.c_size_t, w.DWORD, w.DWORD] + k.VirtualProtect.restype = w.BOOL + k.VirtualProtect.argtypes = [ctypes.c_void_p, ctypes.c_size_t, w.DWORD, ctypes.POINTER(w.DWORD)] + k.FlushInstructionCache.restype = w.BOOL + k.FlushInstructionCache.argtypes = [w.HANDLE, ctypes.c_void_p, ctypes.c_size_t] + k.GetProcessMitigationPolicy.restype = w.BOOL + k.GetProcessMitigationPolicy.argtypes = [w.HANDLE, ctypes.c_int, ctypes.c_void_p, ctypes.c_size_t] + if hasattr(k, 'IsWow64Process2'): # (Windows 10 1709 on) + k.IsWow64Process2.restype = w.BOOL + k.IsWow64Process2.argtypes = [w.HANDLE, ctypes.POINTER(w.USHORT), ctypes.POINTER(w.USHORT)] + return k + + +def _u32(a): + return ctypes.c_uint32.from_address(a).value + + +def _u64(a): + return ctypes.c_uint64.from_address(a).value + + +def _machine(base): + """A loaded module's PE machine (0x8664 x64, 0xAA64 arm64). platform.machine() says ARM64 under emulation.""" + return ctypes.c_uint16.from_address(base + _u32(base + 0x3C) + 4).value + + +def _iat_slot(base, dll, name): + """The address of module base's import-table slot for dll!name, or None.""" + opt = base + _u32(base + 0x3C) + 24 + rva = _u32(opt + 112 + 8) # (data directory 1: imports) + if not rva: + return None + d = base + rva + while _u32(d + 12): # IMAGE_IMPORT_DESCRIPTOR: OriginalFirstThunk, TimeDateStamp, ForwarderChain, Name, FirstThunk + if ctypes.string_at(base + _u32(d + 12)).decode('ascii', 'replace').lower() == dll: + names, slots = base + (_u32(d) or _u32(d + 16)), base + _u32(d + 16) + i = 0 + while True: + t = _u64(names + i * 8) + if not t: + break + if not t >> 63 and ctypes.string_at(base + (t & 0x7FFFFFFF) + 2) == name: + return slots + i * 8 + i += 1 + d += 20 + return None + + +def apply(): + """Point unicorn.dll's setjmp and longjmp at ours (call after `import unicorn`). Returns words for the log.""" + global _stubs + if os.name != 'nt' or struct.calcsize('P') != 8: + return '' + if _stubs: + return 'own setjmp/longjmp' + from ctypes import wintypes as w + k = _k32() + uc = k.GetModuleHandleW('unicorn.dll') + if not uc or _machine(uc) != 0x8664: + return '' + slots = [_iat_slot(uc, 'vcruntime140.dll', n) for n in (b'__intrinsic_setjmp', b'longjmp')] + if not all(slots): # (both or neither: our longjmp reads the frame field our setjmp writes) + return 'setjmp/longjmp not found' + mem = k.VirtualAlloc(None, 4096, 0x3000, 0x04) # (MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE) + if not mem: + return f'setjmp/longjmp not replaced (error {ctypes.get_last_error()})' + ctypes.memmove(mem, SETJMP_X64, len(SETJMP_X64)) + ctypes.memmove(mem + 0x100, LONGJMP_X64, len(LONGJMP_X64)) + old = w.DWORD() + if not k.VirtualProtect(mem, 4096, 0x20, ctypes.byref(old)): # PAGE_EXECUTE_READ + return f'setjmp/longjmp not replaced (error {ctypes.get_last_error()})' + k.FlushInstructionCache(k.GetCurrentProcess(), mem, 4096) + if not k.VirtualProtect(min(slots), max(slots) - min(slots) + 8, 0x04, ctypes.byref(old)): + return f'setjmp/longjmp not replaced (error {ctypes.get_last_error()})' + ctypes.c_uint64.from_address(slots[0]).value = mem + ctypes.c_uint64.from_address(slots[1]).value = mem + 0x100 + k.VirtualProtect(min(slots), max(slots) - min(slots) + 8, old.value, ctypes.byref(old)) + _stubs = mem + return 'own setjmp/longjmp' + + +def _vcruntime_cfg_check(k): + """Whether VCRUNTIME140's CFG check pointer is still its own no-op (the check longjmp skips then).""" + vc = k.GetModuleHandleW('vcruntime140.dll') + if not vc: + return None + opt = vc + _u32(vc + 0x3C) + 24 + size, rva = _u32(opt + 56), _u32(opt + 112 + 10 * 8) # SizeOfImage, data directory 10: load config + if not rva or _u32(vc + rva) < 0x78: + return None + ptr = _u64(vc + rva + 0x70) # GuardCFCheckFunctionPointer + if not ptr: + return None + target = _u64(ptr) + return 'its own' if vc <= target < vc + size else 'set by Windows' + + +def describe(): + """Windows: the CPU this runs on, and whether CFG and shadow stacks are on in this process, for the log.""" + if os.name != 'nt': + return '' + try: + from ctypes import wintypes as w + k = _k32() + me = k.GetCurrentProcess() + machine = {0x8664: 'x64', 0xAA64: 'arm64'} + exe = _machine(k.GetModuleHandleW(None)) + this = machine.get(exe, hex(exe)) + proc, native = w.USHORT(), w.USHORT() + ok = hasattr(k, 'IsWow64Process2') and k.IsWow64Process2(me, ctypes.byref(proc), ctypes.byref(native)) + host = machine.get(native.value, hex(native.value)) if ok else '?' + out = [f'{this} on {host}' + (' (emulated)' if host not in (this, '?') else '')] + for name, policy, bit in (('Control Flow Guard', 7, 1), ('shadow stacks', 17, 1), + ('strict shadow stacks', 17, 0x10)): + flags = w.DWORD() + if k.GetProcessMitigationPolicy(me, policy, ctypes.byref(flags), ctypes.sizeof(flags)): + out.append(f'{name} {"on" if flags.value & bit else "off"}') + else: + out.append(f'{name} unknown (error {ctypes.get_last_error()})') + check = _vcruntime_cfg_check(k) + if check: + out.append(f'VCRUNTIME140 CFG check {check}') + return ', '.join(out) + except Exception as e: # (only for the log) + return f'mitigations unknown ({e})'