test dbcache vs maxsigcache sizing

.github/README.md

@@ -0,0 +1 @@
+../doc/benchcoin.md

.github/workflows/benchmark.yml

@@ -0,0 +1,48 @@
+name: Benchmark
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  benchmark:
+    runs-on: [self-hosted, linux, x64]
+    timeout-minutes: 1000
+    env:
+      ORIGINAL_DATADIR: /data/pruned-840k
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+
+      - name: Run experiment
+        run: |
+          nix develop --command python3 bench.py experiment run \
+            bench/experiments/pr.toml \
+            --datadir $ORIGINAL_DATADIR \
+            --tmp-dir ${{ runner.temp }}/datadirs \
+            --binaries-dir ${{ runner.temp }}/binaries \
+            --output-dir ${{ runner.temp }}/experiment-output \
+            --skip-existing
+
+      - name: Upload experiment output
+        uses: actions/upload-artifact@v7
+        with:
+          name: experiment-output
+          path: ${{ runner.temp }}/experiment-output/
+
+      - name: Write context metadata
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github) }}
+          RUNNER_CONTEXT: ${{ toJSON(runner) }}
+        run: |
+          mkdir -p ${{ runner.temp }}/contexts
+          echo "$GITHUB_CONTEXT" | nix develop --command jq "del(.token)" > ${{ runner.temp }}/contexts/github.json
+          echo "$RUNNER_CONTEXT" > ${{ runner.temp }}/contexts/runner.json
+
+      - name: Upload context metadata
+        uses: actions/upload-artifact@v7
+        with:
+          name: run-metadata
+          path: ${{ runner.temp }}/contexts/

.github/workflows/nightly-benchmark.yml

@@ -0,0 +1,157 @@
+name: Nightly Benchmark
+on:
+  workflow_run:
+    workflows: ["Nightly Rebase"]
+    types: [completed]
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
+    runs-on: [self-hosted, linux, x64]
+    timeout-minutes: 1200
+    env:
+      ORIGINAL_DATADIR: /data/pruned-840k
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 200  # Need enough history for merge-base
+
+      - name: Get commit SHAs
+        run: |
+          # Bitcoin commit - find merge-base with upstream
+          git remote add upstream https://github.com/bitcoin/bitcoin.git
+          git fetch upstream master
+          BITCOIN_SHA=$(git merge-base HEAD upstream/master)
+          echo "BITCOIN_SHA=$BITCOIN_SHA" >> "$GITHUB_ENV"
+
+          # Get commit date for the Bitcoin commit (for chart X-axis)
+          COMMIT_DATE=$(git log -1 --format=%cd --date=short "$BITCOIN_SHA")
+          echo "COMMIT_DATE=$COMMIT_DATE" >> "$GITHUB_ENV"
+
+          echo "Benchcoin: $(git rev-parse HEAD)"
+          echo "Bitcoin merge-base: $BITCOIN_SHA"
+          echo "Commit date: $COMMIT_DATE"
+
+      - name: Run experiment
+        run: |
+          nix develop --command python3 bench.py experiment run \
+            bench/experiments/nightly.toml \
+            --datadir $ORIGINAL_DATADIR \
+            --tmp-dir ${{ runner.temp }}/datadirs \
+            --binaries-dir ${{ runner.temp }}/binaries \
+            --output-dir ${{ runner.temp }}/experiment-output \
+            --skip-existing
+
+      - name: Upload experiment output
+        uses: actions/upload-artifact@v7
+        with:
+          name: nightly-experiment-output
+          path: ${{ runner.temp }}/experiment-output/
+
+      - name: Upload commit info
+        run: |
+          echo "$BITCOIN_SHA" > ${{ runner.temp }}/commit.txt
+          echo "$COMMIT_DATE" > ${{ runner.temp }}/commit-date.txt
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: commit-info
+          path: |
+            ${{ runner.temp }}/commit.txt
+            ${{ runner.temp }}/commit-date.txt
+
+      - name: Capture machine specs
+        run: |
+          nix develop --command python3 -c "
+          from bench.machine import get_machine_specs
+          import json
+          print(json.dumps(get_machine_specs().to_dict()))
+          " > ${{ runner.temp }}/machine-specs.json
+
+      - name: Upload machine specs
+        uses: actions/upload-artifact@v7
+        with:
+          name: machine-specs
+          path: ${{ runner.temp }}/machine-specs.json
+
+  publish:
+    needs: benchmark
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout gh-pages
+        uses: actions/checkout@v6
+        with:
+          ref: gh-pages
+
+      - name: Checkout benchcoin tools
+        uses: actions/checkout@v6
+        with:
+          ref: master
+          path: benchcoin-tools
+
+      - name: Download commit info
+        uses: actions/download-artifact@v7
+        with:
+          name: commit-info
+          path: ./commit-info
+
+      - name: Download experiment output
+        uses: actions/download-artifact@v7
+        with:
+          name: nightly-experiment-output
+          path: ./nightly-experiment-output
+
+      - name: Download machine specs
+        uses: actions/download-artifact@v7
+        with:
+          name: machine-specs
+          path: ./machine-specs
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+
+      - name: Get dates and trigger
+        run: |
+          # Commit date (for chart X-axis)
+          COMMIT_DATE=$(cat ./commit-info/commit-date.txt)
+          echo "COMMIT_DATE=$COMMIT_DATE" >> "$GITHUB_ENV"
+          # Run date (for reference)
+          echo "RUN_DATE=$(date -u +%Y-%m-%d)" >> "$GITHUB_ENV"
+          # Trigger type (scheduled nightly vs manual dispatch)
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "TRIGGER=manual" >> "$GITHUB_ENV"
+          else
+            echo "TRIGGER=scheduled" >> "$GITHUB_ENV"
+          fi
+
+      - name: Append results to history
+        run: |
+          COMMIT=$(cat ./commit-info/commit.txt)
+          nix develop ./benchcoin-tools --command python3 benchcoin-tools/bench.py nightly \
+            --history-file ./nightly-history.json \
+            append-experiment \
+            ./nightly-experiment-output \
+            "$COMMIT" \
+            --date "$COMMIT_DATE" \
+            --run-date "$RUN_DATE" \
+            --trigger "$TRIGGER" \
+            --machine-specs ./machine-specs/machine-specs.json
+
+      - name: Generate chart
+        run: |
+          nix develop ./benchcoin-tools --command python3 benchcoin-tools/bench.py nightly \
+            --history-file ./nightly-history.json \
+            chart \
+            ./index.html
+
+      - name: Commit and push to gh-pages
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add nightly-history.json index.html
+          git commit -m "Update nightly benchmark results for $COMMIT_DATE" || echo "No changes to commit"
+          git push origin gh-pages

.github/workflows/publish-results.yml

@@ -0,0 +1,139 @@
+name: Publish Results
+on:
+  workflow_run:
+    workflows: ["Benchmark"]
+    types: [completed]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    permissions:
+      actions: read
+      contents: write
+      checks: read
+    outputs:
+      comparison: ${{ steps.generate.outputs.comparison }}
+      pr-number: ${{ steps.metadata.outputs.pr-number }}
+      result-url: ${{ steps.generate.outputs.result-url }}
+      pages-commit: ${{ steps.push-pages.outputs.pages-commit }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: gh-pages
+
+      - name: Checkout benchcoin tools
+        uses: actions/checkout@v6
+        with:
+          ref: master
+          path: benchcoin-tools
+
+      - name: Download artifacts
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh run download ${{ github.event.workflow_run.id }} --repo ${{ github.repository }}
+
+      - name: Extract metadata
+        run: |
+          if [ -d "run-metadata" ]; then
+            mkdir -p run-metadata-extracted
+            mv run-metadata/* run-metadata-extracted/
+          fi
+
+      - name: Read metadata
+        id: metadata
+        run: |
+          PR_NUMBER=$(jq -r '.event.pull_request.number // "main"' run-metadata-extracted/github.json)
+          RUN_ID=$(jq -r '.run_id' run-metadata-extracted/github.json)
+          HEAD_SHA=$(jq -r '.event.pull_request.head.sha // .sha' run-metadata-extracted/github.json)
+          echo "pr-number=${PR_NUMBER}" >> $GITHUB_OUTPUT
+          echo "run-id=${RUN_ID}" >> $GITHUB_OUTPUT
+          echo "head-sha=${HEAD_SHA}" >> $GITHUB_OUTPUT
+          echo "Found metadata: PR=${PR_NUMBER}, Run=${RUN_ID}, Commit=${HEAD_SHA}"
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+
+      - name: Generate report
+        id: generate
+        env:
+          PR_NUMBER: ${{ steps.metadata.outputs.pr-number }}
+          RUN_ID: ${{ steps.metadata.outputs.run-id }}
+          HEAD_SHA: ${{ steps.metadata.outputs.head-sha }}
+        run: |
+          # Generate report with nightly comparison (use per-machine history file)
+          nix develop ./benchcoin-tools --command python3 benchcoin-tools/bench.py report \
+            --experiment-output "./experiment-output" \
+            --pr-number "${PR_NUMBER}" \
+            --run-id "${RUN_ID}" \
+            --commit "${HEAD_SHA}" \
+            --nightly-history "./nightly-history.json" \
+            --update-index \
+            "./results/pr-${PR_NUMBER}/${RUN_ID}"
+
+          COMPARISON=$(cat "./results/pr-${PR_NUMBER}/${RUN_ID}/summary.txt")
+          echo "comparison<<EOF" >> $GITHUB_OUTPUT
+          echo "${COMPARISON}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+          RESULT_URL="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/results/pr-${PR_NUMBER}/${RUN_ID}/index.html"
+          echo "result-url=${RESULT_URL}" >> $GITHUB_OUTPUT
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: results
+
+      - name: Commit and push to gh-pages
+        id: push-pages
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          # Note: Only add results/ directory, not root index.html
+          # The root index.html is managed by nightly-benchmark.yml
+          git add results/
+          git commit -m "Update benchmark results from run ${{ github.event.workflow_run.id }}"
+          git push origin gh-pages
+          echo "pages-commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+
+  wait-for-pages:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for GitHub Pages deployment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          COMMIT="${{ needs.build.outputs.pages-commit }}"
+          for i in $(seq 1 60); do
+            RUN_ID=$(gh run list -R "${{ github.repository }}" -w "pages-build-deployment" --commit "$COMMIT" -L 1 --json databaseId -q '.[0].databaseId')
+            if [ -n "$RUN_ID" ]; then
+              echo "Found pages-build-deployment run ${RUN_ID} for commit ${COMMIT}"
+              gh run watch "$RUN_ID" -R "${{ github.repository }}"
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "::warning::Could not find pages-build-deployment run for commit ${COMMIT}"
+
+  comment-pr:
+    needs: [build, wait-for-pages]
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Comment on PR
+        if: ${{ needs.build.outputs.pr-number != 'main' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh pr comment ${{ needs.build.outputs.pr-number }} \
+            --repo ${{ github.repository }} \
+            --body "## Benchmark Results
+
+          **Comparison to nightly master (median of last 7 runs):**
+          - ${{ needs.build.outputs.comparison }}
+
+          [View detailed results](${{ needs.build.outputs.result-url }})
+          [View nightly trend chart](https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/)"

.github/workflows/rebase.yml

@@ -0,0 +1,35 @@
+name: Nightly Rebase
+
+on:
+  schedule:
+    - cron: '0 1 * * *'  # 01:00 GMT daily
+  workflow_dispatch:  # manual trigger
+
+permissions:
+  contents: write
+
+jobs:
+  rebase:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          ref: master
+          fetch-depth: 0  # Full history needed for rebase
+          token: ${{ secrets.REBASE_PAT }}
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Add upstream and rebase
+        run: |
+          git remote add upstream https://github.com/bitcoin/bitcoin.git
+          git fetch upstream master
+          git rebase upstream/master
+
+      - name: Push changes
+        run: |
+          git push --force origin master