Recover completed P1 artifacts without re-running collection

.github/workflows/rhc-canonical-chain-history.yml

@@ -0,0 +1,199 @@
+name: RHC canonical SeaDrop and Seaport history
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+    paths:
+      - '.github/workflows/rhc-canonical-chain-history.yml'
+      - 'chain-history/**'
+      - 'chain-history-trigger.txt'
+
+permissions:
+  actions: read
+  contents: write
+
+concurrency:
+  group: rhc-canonical-chain-history-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  head:
+    runs-on: ubuntu-latest
+    outputs:
+      head: ${{ steps.capture.outputs.head }}
+    steps:
+      - id: capture
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY' >> "$GITHUB_OUTPUT"
+          import json, urllib.request
+          payload=json.dumps({'jsonrpc':'2.0','id':1,'method':'eth_blockNumber','params':[]}).encode()
+          request=urllib.request.Request('https://rpc.mainnet.chain.robinhood.com',data=payload,headers={'Content-Type':'application/json','User-Agent':'rhc-head-capture/1.0'})
+          with urllib.request.urlopen(request,timeout=60) as response:
+              data=json.loads(response.read())
+          current=int(data['result'],16)
+          finalized=max(0,current-1000)
+          print(f'head={finalized}')
+          PY
+
+  collect:
+    needs: head
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        event: [seadrop, seaport]
+        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Collect canonical event shard
+        shell: bash
+        run: |
+          set -euo pipefail
+          python chain-history/collect_logs.py \
+            --event '${{ matrix.event }}' \
+            --shard '${{ matrix.shard }}' \
+            --shards 64 \
+            --head '${{ needs.head.outputs.head }}' \
+            --out 'out-${{ matrix.event }}-${{ matrix.shard }}'
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: rhc-chain-${{ matrix.event }}-${{ matrix.shard }}
+          path: out-${{ matrix.event }}-${{ matrix.shard }}/
+          retention-days: 3
+          if-no-files-found: error
+          compression-level: 9
+
+  aggregate:
+    needs: [head, collect]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: chatgpt/rhc-wallet-verification-p0p1-20260828
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: rhc-chain-*
+          path: downloaded-chain-history
+      - name: Merge shards and enforce exact coverage
+        env:
+          FIXED_HEAD: ${{ needs.head.outputs.head }}
+          COLLECT_RESULT: ${{ needs.collect.result }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY'
+          import gzip, hashlib, json, os, pathlib, shutil
+
+          head=int(os.environ['FIXED_HEAD'])
+          collect_result=os.environ['COLLECT_RESULT']
+          source=pathlib.Path('downloaded-chain-history')
+          target=pathlib.Path('canonical-chain-history')
+          if target.exists():
+              shutil.rmtree(target)
+          target.mkdir()
+
+          overall={'fixed_head':head,'matrix_result':collect_result,'events':{},'production_approved_wallets':0}
+          global_failures=[]
+          for event in ('seadrop','seaport'):
+              event_dir=target/event
+              event_dir.mkdir()
+              logs={}
+              ranges=[]
+              shard_validations=[]
+              missing=[]
+              for shard in range(64):
+                  d=source/f'rhc-chain-{event}-{shard}'
+                  if not d.exists():
+                      missing.append(shard)
+                      continue
+                  validation_path=d/'validation.json'
+                  coverage_path=d/'coverage.json'
+                  log_path=d/'logs.jsonl.gz'
+                  if not validation_path.exists() or not coverage_path.exists() or not log_path.exists():
+                      missing.append(shard)
+                      continue
+                  validation=json.loads(validation_path.read_text(encoding='utf-8'))
+                  coverage=json.loads(coverage_path.read_text(encoding='utf-8'))
+                  shard_validations.append(validation)
+                  ranges.append((coverage['start'],coverage['end'],shard))
+                  with gzip.open(log_path,'rt',encoding='utf-8') as handle:
+                      for line in handle:
+                          row=json.loads(line)
+                          key=(row.get('blockHash'),row.get('transactionHash'),row.get('logIndex'))
+                          logs[key]=row
+              failures=[]
+              if missing:
+                  failures.append({'code':'MISSING_SHARDS','shards':missing})
+              if any(row.get('status')!='PASS' for row in shard_validations):
+                  failures.append({'code':'SHARD_VALIDATION_FAILED','count':sum(row.get('status')!='PASS' for row in shard_validations)})
+              ranges.sort()
+              expected_start=0
+              for start,end,shard in ranges:
+                  if start!=expected_start:
+                      failures.append({'code':'COVERAGE_GAP_OR_OVERLAP','expected_start':expected_start,'actual_start':start,'shard':shard})
+                  expected_start=end+1
+              if expected_start!=head+1:
+                  failures.append({'code':'HEAD_NOT_COVERED','expected_end':head,'actual_end':expected_start-1})
+
+              ordered=sorted(logs.values(),key=lambda row:(int(row['blockNumber'],16),int(row.get('transactionIndex') or '0x0',16),int(row['logIndex'],16)))
+              out=event_dir/f'{event}_logs.jsonl.gz'
+              with gzip.open(out,'wt',encoding='utf-8',newline='\n') as handle:
+                  for row in ordered:
+                      handle.write(json.dumps(row,ensure_ascii=False,sort_keys=True)+'\n')
+              validation={
+                  'status':'PASS' if not failures else 'FAIL',
+                  'event':event,
+                  'fixed_head':head,
+                  'shards_expected':64,
+                  'shards_found':len(ranges),
+                  'unique_logs':len(ordered),
+                  'failures':failures,
+                  'matrix_result':collect_result,
+              }
+              (event_dir/'VALIDATION.json').write_text(json.dumps(validation,ensure_ascii=False,indent=2,sort_keys=True),encoding='utf-8')
+              overall['events'][event]=validation
+              global_failures.extend({'event':event,**failure} for failure in failures)
+
+          overall['status']='PASS' if not global_failures and collect_result=='success' else 'FAIL'
+          overall['failures']=global_failures
+          (target/'VALIDATION.json').write_text(json.dumps(overall,ensure_ascii=False,indent=2,sort_keys=True),encoding='utf-8')
+
+          manifest=[]
+          for p in sorted(target.rglob('*')):
+              if p.is_file():
+                  manifest.append({'path':str(p.relative_to(target)),'bytes':p.stat().st_size,'sha256':hashlib.sha256(p.read_bytes()).hexdigest()})
+          (target/'MANIFEST.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2,sort_keys=True),encoding='utf-8')
+          print(json.dumps(overall,sort_keys=True))
+          PY
+      - name: Commit raw canonical history and validation
+        shell: bash
+        run: |
+          set -euo pipefail
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git add canonical-chain-history
+          if ! git diff --cached --quiet; then
+            git commit -m 'data: persist canonical SeaDrop and Seaport histories'
+            git push origin HEAD:chatgpt/rhc-wallet-verification-p0p1-20260828
+          fi
+          python - <<'PY'
+          import json
+          data=json.load(open('canonical-chain-history/VALIDATION.json'))
+          if data['status']!='PASS':
+              raise SystemExit(1)
+          PY

.github/workflows/rhc-finalize-p1.yml

@@ -0,0 +1,148 @@
+name: Finalize RHC P1 verification checkpoint
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+    paths:
+      - '.github/workflows/rhc-finalize-p1.yml'
+      - 'recovered-p1-partial-checkpoint/RECOVERY_REPORT.json'
+
+permissions:
+  actions: read
+  contents: write
+
+concurrency:
+  group: rhc-finalize-p1-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  finalize:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      GH_TOKEN: ${{ github.token }}
+      TARGET_WORKFLOW_NAME: 'Re-run only missing RHC P1 shards'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          fetch-depth: 1
+      - name: Determine whether finalization is ready
+        id: plan
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY' >> "$GITHUB_OUTPUT"
+          import json
+          from pathlib import Path
+          p=Path('recovered-p1-partial-checkpoint/RECOVERY_REPORT.json')
+          if not p.exists():
+              print('ready=false')
+              print('has_missing=false')
+              print('missing_json=[]')
+          else:
+              d=json.loads(p.read_text())
+              missing=[k for k,v in d.get('shards',{}).items() if v.get('status')!='RECOVERED_NONEMPTY']
+              print('ready=true')
+              print('has_missing='+('true' if missing else 'false'))
+              print('missing_json='+json.dumps(missing,separators=(',',':')))
+          PY
+      - name: Stop cleanly until recovery report exists
+        if: steps.plan.outputs.ready != 'true'
+        run: echo 'Recovery report is not present yet; no data collection or re-run was started.'
+      - name: Reconstruct recovered partial package
+        if: steps.plan.outputs.ready == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          cat recovered-p1-partial-checkpoint/rhc-p1-partial-recovered.tar.gz.part-* > rhc-p1-partial-recovered.tar.gz
+          expected=$(awk '{print $1}' recovered-p1-partial-checkpoint/SHA256SUMS.txt)
+          actual=$(sha256sum rhc-p1-partial-recovered.tar.gz | awk '{print $1}')
+          test "$expected" = "$actual"
+          rm -rf final-p1-work
+          mkdir final-p1-work
+          tar -xzf rhc-p1-partial-recovered.tar.gz -C final-p1-work
+      - name: Wait for and download targeted missing-shard artifacts
+        if: steps.plan.outputs.ready == 'true' && steps.plan.outputs.has_missing == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          branch='${{ github.event.pull_request.head.ref }}'
+          run_id=''
+          for i in $(seq 1 180); do
+            run_id=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?branch=${branch}&per_page=100" --jq ".workflow_runs | map(select(.name == env.TARGET_WORKFLOW_NAME)) | sort_by(.id) | last | .id // empty")
+            if [ -n "$run_id" ]; then
+              status=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" --jq '.status')
+              conclusion=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" --jq '.conclusion // ""')
+              echo "targeted run ${run_id}: ${status} ${conclusion}"
+              if [ "$status" = completed ]; then
+                test "$conclusion" = success
+                break
+              fi
+            fi
+            sleep 15
+          done
+          test -n "$run_id"
+          mkdir -p final-p1-work/targeted-reruns
+          gh run download "$run_id" -R "$GITHUB_REPOSITORY" -D final-p1-work/targeted-reruns
+          echo "$run_id" > final-p1-work/TARGETED_RERUN_ID.txt
+      - name: Build final four-shard evidence checkpoint
+        if: steps.plan.outputs.ready == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY'
+          from pathlib import Path
+          import csv,json,hashlib,shutil
+          root=Path('final-p1-work')
+          candidates=[p for p in root.rglob('*') if p.is_file()]
+          report={'status':'PASS','shards':{}}
+          for shard in ('p1a','p1b','p1c','p1d'):
+              paths=[p for p in candidates if shard in str(p).lower()]
+              counts={};substantive=0
+              for p in paths:
+                  if p.suffix.lower()=='.csv':
+                      try:
+                          with p.open(newline='',encoding='utf-8-sig') as f:n=sum(1 for _ in csv.DictReader(f))
+                          counts[str(p)]=n
+                          if n>0 and any(k in p.name.lower() for k in ('transaction','transfer','receipt','event','evidence','summary','wallet')):substantive+=n
+                      except Exception as exc:counts[str(p)]={'error':repr(exc)}
+                  elif p.suffix.lower()=='.json':
+                      try:
+                          obj=json.loads(p.read_text(encoding='utf-8-sig'));counts[str(p)]={'json_type':type(obj).__name__}
+                      except Exception as exc:counts[str(p)]={'error':repr(exc)}
+              status='PASS_NONEMPTY' if paths and substantive>0 else 'FAIL_EMPTY_OR_MISSING'
+              if status!='PASS_NONEMPTY':report['status']='FAIL'
+              report['shards'][shard]={'status':status,'file_count':len(paths),'substantive_rows':substantive,'row_counts':counts}
+          report['production_approved_wallets']=0
+          report['decision_use']='RESEARCH_EVIDENCE_ONLY_NOT_STRONG_WALLET_APPROVAL'
+          Path('final-p1-work/P1_FINAL_VALIDATION.json').write_text(json.dumps(report,indent=2,sort_keys=True),encoding='utf-8')
+          print(json.dumps(report,indent=2,sort_keys=True))
+          if report['status']!='PASS':raise SystemExit(1)
+          # immutable manifest over final work before packaging
+          rows=[]
+          for p in sorted(root.rglob('*')):
+              if p.is_file():rows.append({'path':str(p.relative_to(root)),'bytes':p.stat().st_size,'sha256':hashlib.sha256(p.read_bytes()).hexdigest()})
+          Path('final-p1-work/MANIFEST.json').write_text(json.dumps(rows,indent=2,sort_keys=True),encoding='utf-8')
+          PY
+          tar -czf rhc-p1-final.tar.gz final-p1-work
+          rm -rf recovered-p1-final-checkpoint
+          mkdir recovered-p1-final-checkpoint
+          split -b 40m -d -a 3 rhc-p1-final.tar.gz recovered-p1-final-checkpoint/rhc-p1-final.tar.gz.part-
+          sha256sum rhc-p1-final.tar.gz > recovered-p1-final-checkpoint/SHA256SUMS.txt
+          cp final-p1-work/P1_FINAL_VALIDATION.json final-p1-work/MANIFEST.json recovered-p1-final-checkpoint/
+      - name: Commit final P1 checkpoint to temporary branch
+        if: steps.plan.outputs.ready == 'true'
+        shell: bash
+        run: |
+          set -euo pipefail
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git add recovered-p1-final-checkpoint
+          if git diff --cached --quiet; then
+            echo 'No final P1 checkpoint changes.'
+          else
+            git commit -m 'chore: finalize complete P1 verification checkpoint'
+            git push origin "HEAD:${{ github.event.pull_request.head.ref }}"
+          fi