Improve dump fidelity, system performance, security, and recovery

.github/scripts/release_artifacts.py

@@ -0,0 +1,172 @@
+"""Copy only the expected binary bytes from this run's untrusted artifacts."""
+
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import re
+import stat
+from urllib.error import HTTPError
+from urllib.parse import urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+import zipfile
+
+
+MAX_BYTES = 64 * 1024 * 1024
+ARTIFACTS = {"Win32 Executable": "pd32.exe", "x64 Executable": "pd64.exe"}
+
+
+class NoRedirect(HTTPRedirectHandler):
+    def redirect_request(self, request, fp, code, message, headers, new_url):
+        return None
+
+
+def bounded_read(response, limit):
+    data = response.read(limit + 1)
+    if len(data) > limit:
+        raise ValueError("Response exceeds the release size limit")
+    return data
+
+
+class GitHub:
+    def __init__(self):
+        self.base = os.environ["GITHUB_API_URL"].rstrip("/")
+        self.token = os.environ["GH_TOKEN"]
+        self.opener = build_opener(NoRedirect())
+        if urlsplit(self.base).scheme != "https":
+            raise ValueError("The GitHub API must use HTTPS")
+
+    def request(self, endpoint):
+        return Request(self.base + endpoint, headers={
+            "Authorization": "Bearer " + self.token,
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        })
+
+    def json(self, endpoint, allow_missing=False):
+        try:
+            with self.opener.open(self.request(endpoint), timeout=60) as response:
+                return json.loads(bounded_read(response, 4 * 1024 * 1024))
+        except HTTPError as error:
+            if error.code == 404 and allow_missing:
+                return None
+            raise
+
+    def archive(self, repository, artifact_id):
+        endpoint = f"/repos/{repository}/actions/artifacts/{artifact_id}/zip"
+        try:
+            with self.opener.open(self.request(endpoint), timeout=60) as response:
+                return bounded_read(response, MAX_BYTES)
+        except HTTPError as error:
+            if error.code != 302:
+                raise
+            location = error.headers["Location"]
+        parsed = urlsplit(location)
+        if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
+            raise ValueError("Invalid artifact download redirect")
+        # Never forward the repository token to the signed blob URL or follow another redirect.
+        with self.opener.open(Request(location), timeout=60) as response:
+            return bounded_read(response, MAX_BYTES)
+
+
+def select_artifacts(api, repository, run_id):
+    selected = {}
+    for page in range(1, 101):
+        listing = api.json(f"/repos/{repository}/actions/runs/{run_id}/artifacts?per_page=100&page={page}")
+        entries = listing["artifacts"]
+        for item in entries:
+            name = item["name"]
+            if name not in ARTIFACTS:
+                continue
+            if name in selected or item["expired"]:
+                raise ValueError("Duplicate or expired release artifact")
+            if type(item["id"]) is not int or item["id"] <= 0:
+                raise ValueError("Invalid release artifact ID")
+            if not 0 < item["size_in_bytes"] <= MAX_BYTES:
+                raise ValueError("Invalid release artifact size")
+            selected[name] = item
+        if len(entries) < 100:
+            break
+    else:
+        raise ValueError("Too many artifacts in the release run")
+    if selected.keys() != ARTIFACTS.keys():
+        raise ValueError("Both architecture artifacts are required")
+    return selected
+
+
+def validate_entry(entries):
+    if len(entries) != 1:
+        raise ValueError("A release artifact must contain exactly one entry")
+    entry = entries[0]
+    mode = entry.external_attr >> 16
+    if (entry.filename != "pd.exe" or entry.orig_filename != "pd.exe" or entry.is_dir()
+            or stat.S_IFMT(mode) not in (0, stat.S_IFREG)
+            or entry.external_attr & 0x10 or entry.flag_bits & 1
+            or entry.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED)
+            or not 0 < entry.file_size <= MAX_BYTES
+            or not 0 <= entry.compress_size <= MAX_BYTES):
+        raise ValueError("Unexpected release archive entry")
+    return entry
+
+
+def copy_binary(archive, destination):
+    if len(archive) > MAX_BYTES:
+        raise ValueError("Release archive exceeds the size limit")
+    with zipfile.ZipFile(io.BytesIO(archive)) as zipped:
+        entry = validate_entry(zipped.infolist())
+        # Validate data/CRC before writing; archive-controlled paths are never opened.
+        with zipped.open(entry) as source:
+            data = bounded_read(source, MAX_BYTES)
+        if len(data) != entry.file_size:
+            raise ValueError("Release binary length mismatch")
+    with destination.open("xb") as output:
+        output.write(data)
+    return {"size": len(data), "sha256": hashlib.sha256(data).hexdigest()}
+
+
+def verify_tag(api, repository, version, commit):
+    ref = api.json(f"/repos/{repository}/git/ref/tags/{version}", allow_missing=True)
+    if ref is None:
+        return
+    target = ref["object"]
+    for _ in range(8):
+        if target["type"] == "commit":
+            if target["sha"].lower() != commit.lower():
+                raise ValueError("Existing version tag does not identify the tested commit")
+            return
+        if target["type"] != "tag" or not re.fullmatch(r"[0-9a-fA-F]{40}", target["sha"]):
+            break
+        target = api.json(f"/repos/{repository}/git/tags/{target['sha']}")["object"]
+    raise ValueError("Unsupported release tag target")
+
+
+def main():
+    repository = os.environ["GITHUB_REPOSITORY"]
+    run_id = os.environ["GITHUB_RUN_ID"]
+    commit = os.environ["RELEASE_COMMIT"]
+    version = os.environ["RELEASE_VERSION"]
+    if (not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository)
+            or not re.fullmatch(r"[0-9]+", run_id)
+            or not re.fullmatch(r"[0-9a-fA-F]{40}", commit)
+            or not (version == "Develop" or re.fullmatch(r"v[0-9]+\.[0-9]+(\.[0-9]+)?", version))):
+        raise ValueError("Invalid release identity")
+    api = GitHub()
+    if version != "Develop":
+        verify_tag(api, repository, version, commit)
+    selected = select_artifacts(api, repository, run_id)
+    output = Path("release-binaries")
+    output.mkdir()
+    manifest = {"repository": repository, "commit": commit, "run_id": run_id, "files": {}}
+    for name, filename in ARTIFACTS.items():
+        artifact = selected[name]
+        result = copy_binary(api.archive(repository, artifact["id"]), output / filename)
+        result["artifact_id"] = artifact["id"]
+        manifest["files"][filename] = result
+    with (output / "provenance.json").open("x", encoding="utf-8") as destination:
+        json.dump(manifest, destination, indent=2)
+        destination.write("\n")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/build.yml

@@ -22,17 +22,22 @@ jobs:
         platform: [Win32, x64]
         configuration: [Debug, Release]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           ref: ${{ inputs.ref || github.sha }}
-      - uses: microsoft/setup-msbuild@v2
+          persist-credentials: false
+      - uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce # v2
       - name: Build application and unit tests
         run: msbuild pd.sln /m /v:minimal /p:Configuration=${{ matrix.configuration }} /p:Platform=${{ matrix.platform }}
       - name: Run unit tests
         run: |
           & ".\${{ matrix.platform }}\${{ matrix.configuration }}\pd_tests.exe"
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-      - uses: actions/upload-artifact@v4
+      - name: Test CLI documentation, benchmark analysis and release artifact handling
+        env:
+          PD_TEST_EXE: .\${{ matrix.platform }}\${{ matrix.configuration }}\pd.exe
+        run: python -m unittest discover -s tests -p "test_*.py"
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         if: matrix.configuration == 'Release'
         with:
           name: ${{ matrix.platform }} Executable

.github/workflows/deploy-create-releases.yml

@@ -9,7 +9,7 @@ on:
         required: true
       version:
         type: string
-        description: Release version (for example v2.2)
+        description: Release version (for example v3.0.0)
         required: true
 
 permissions:
@@ -35,24 +35,29 @@ jobs:
     needs: build
     runs-on: windows-2022
     permissions:
+      actions: read
       contents: write
+    concurrency:
+      group: publish-${{ inputs.version }}
+      cancel-in-progress: false
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          name: Win32 Executable
-          path: binaries\Win32
-      - uses: actions/download-artifact@v4
-        with:
-          name: x64 Executable
-          path: binaries\x64
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+          sparse-checkout: .github/scripts
+      - name: Prepare allowlisted artifact bytes
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_VERSION: ${{ inputs.version }}
+          RELEASE_COMMIT: ${{ inputs.commit }}
+        run: python -I .github\scripts\release_artifacts.py
       - name: Publish tested binaries
         env:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           RELEASE_VERSION: ${{ inputs.version }}
           RELEASE_COMMIT: ${{ inputs.commit }}
         run: |
-          Move-Item binaries\Win32\pd.exe pd32.exe
-          Move-Item binaries\x64\pd.exe pd64.exe
-          gh release create $env:RELEASE_VERSION pd32.exe pd64.exe --target $env:RELEASE_COMMIT --title $env:RELEASE_VERSION --generate-notes
+          gh release create $env:RELEASE_VERSION release-binaries\pd32.exe release-binaries\pd64.exe release-binaries\provenance.json --target $env:RELEASE_COMMIT --title $env:RELEASE_VERSION --generate-notes
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

.github/workflows/deploy-develop.yml

@@ -15,28 +15,45 @@ jobs:
     needs: build
     runs-on: windows-2022
     permissions:
+      actions: read
       contents: write
+    concurrency:
+      group: publish-Develop
+      cancel-in-progress: false
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          name: Win32 Executable
-          path: binaries\Win32
-      - uses: actions/download-artifact@v4
-        with:
-          name: x64 Executable
-          path: binaries\x64
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+          sparse-checkout: .github/scripts
+      - name: Prepare allowlisted artifact bytes
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_VERSION: Develop
+          RELEASE_COMMIT: ${{ github.sha }}
+        run: python -I .github\scripts\release_artifacts.py
       - name: Publish tested development binaries
         env:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           RELEASE_COMMIT: ${{ github.sha }}
         run: |
-          Move-Item binaries\Win32\pd.exe pd32.exe
-          Move-Item binaries\x64\pd.exe pd64.exe
-          gh release view Develop
-          if ($LASTEXITCODE -eq 0) {
-            gh release upload Develop pd32.exe pd64.exe --clobber
+          if ($env:GITHUB_EVENT_NAME -eq "push") {
+            $current = gh api "repos/$env:GH_REPO/git/ref/heads/main" --jq .object.sha
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+            if ($current -ne $env:RELEASE_COMMIT) {
+              Write-Host "Skipping superseded main build $env:RELEASE_COMMIT"
+              exit 0
+            }
+          }
+          $release = gh api --paginate "repos/$env:GH_REPO/releases?per_page=100" --jq '.[] | select(.tag_name == "Develop") | .id'
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          $notes = "Development binaries from $env:RELEASE_COMMIT. Verify both assets against provenance.json; the historical Develop tag is not moved."
+          if ($release) {
+            gh release upload Develop release-binaries\pd32.exe release-binaries\pd64.exe release-binaries\provenance.json --clobber
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+            gh release edit Develop --notes $notes
           } else {
-            gh release create Develop pd32.exe pd64.exe --target $env:RELEASE_COMMIT --title Develop --prerelease --notes "Development binaries"
+            gh release create Develop release-binaries\pd32.exe release-binaries\pd64.exe release-binaries\provenance.json --target $env:RELEASE_COMMIT --title Develop --prerelease --notes $notes
           }
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

.github/workflows/deploy-releases.yml

@@ -14,23 +14,28 @@ jobs:
     needs: build
     runs-on: windows-2022
     permissions:
+      actions: read
       contents: write
+    concurrency:
+      group: publish-${{ github.ref_name }}
+      cancel-in-progress: false
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
-          name: Win32 Executable
-          path: binaries\Win32
-      - uses: actions/download-artifact@v4
-        with:
-          name: x64 Executable
-          path: binaries\x64
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+          sparse-checkout: .github/scripts
+      - name: Prepare allowlisted artifact bytes
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_VERSION: ${{ github.ref_name }}
+          RELEASE_COMMIT: ${{ github.sha }}
+        run: python -I .github\scripts\release_artifacts.py
       - name: Publish tested release binaries
         env:
           GH_TOKEN: ${{ github.token }}
           GH_REPO: ${{ github.repository }}
           RELEASE_VERSION: ${{ github.ref_name }}
         run: |
-          Move-Item binaries\Win32\pd.exe pd32.exe
-          Move-Item binaries\x64\pd.exe pd64.exe
-          gh release create $env:RELEASE_VERSION pd32.exe pd64.exe --verify-tag --title $env:RELEASE_VERSION --generate-notes
+          gh release create $env:RELEASE_VERSION release-binaries\pd32.exe release-binaries\pd64.exe release-binaries\provenance.json --verify-tag --title $env:RELEASE_VERSION --generate-notes
           if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

.gitignore

@@ -394,3 +394,6 @@ FakesAssemblies/
 # Visual Studio 6 workspace options file
 *.opt
 >>>>>>> d26ace875aaef086a6992374a3169ca5c6db031d
+
+# Python benchmark analysis
+__pycache__