diff --git a/.gitignore b/.gitignore index 3f51d47..1eb9009 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,4 @@ tools/_toolchain_cache/ /dist/ /owner-playtest/ /launch-temp/ +jp/saves/ diff --git a/eu/seamless_catalog.py b/eu/seamless_catalog.py index 3e2d6b7..51aec1f 100644 --- a/eu/seamless_catalog.py +++ b/eu/seamless_catalog.py @@ -20,25 +20,27 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--oracle', action='store_true', help='Compare unique streams with the original EU MIPS decoder, two fills') - parser.add_argument('--variant',choices=['eu1','eu2'],default='eu1') + parser.add_argument('--variant',choices=['eu1','eu2','jp'],default='eu1') + parser.add_argument('--output',type=Path,help='Private generated catalog directory') parser.add_argument('--oracle-limit',type=int,help='Explicit bounded original-MIPS sample; never claims full corpus') args = parser.parse_args() started = time.monotonic() - cue = ROOT/('disc/eu2/disc.cue' if args.variant=='eu2' else 'disc/Tombi! (Europe) (En,Fr,De,It,Nl) (Rev 1).cue') + cue = ROOT/('disc/jp/disc.cue' if args.variant=='jp' else 'disc/eu2/disc.cue' if args.variant=='eu2' else 'disc/Tombi! (Europe) (En,Fr,De,It,Nl) (Rev 1).cue') disc = Disc(cue) eu2_profile = None if args.variant=='eu2': sys.path.insert(0,str(ROOT/'eu2')) from revisions import PROFILES, active, source eu2_profile=PROFILES[active()] - check_disc(dict(disc_hashes={'sha1': eu2_profile['sha1'] if eu2_profile else + check_disc(dict(disc_hashes={'sha1': 'c86baf702303e7e0963b99aff307b1c40e40478a' if args.variant=='jp' else eu2_profile['sha1'] if eu2_profile else '2505d958b4439d04844cadd3f968fc7432a48e00'}), disc, cue) oracle = None if args.oracle: - sys.path.insert(0, str(ROOT/('build-eu2-seamless/python-deps' if args.variant=='eu2' else 'build-eu-seamless/python-deps'))) - oracle = DecoderOracle(disc.read('EXE/MAIN0.EXE' if args.variant=='eu2' else 'EXE/MAIN2.EXE'), - entry=int(json.loads(source('seamless-contract.json').read_text())['languages'][0]['code']['0x8003bafc']['address'],16) if eu2_profile else 0x8003BAFC) - out = ROOT/('build-eu2-seamless/generated' if args.variant=='eu2' else 'build-eu-seamless/generated') + sys.path.insert(0, str(ROOT/('build-jp/full/test-deps' if args.variant=='jp' else 'build-eu2-seamless/python-deps' if args.variant=='eu2' else 'build-eu-seamless/python-deps'))) + oracle = DecoderOracle(disc.read('SLPS_011.44' if args.variant=='jp' else 'EXE/MAIN0.EXE' if args.variant=='eu2' else 'EXE/MAIN2.EXE'), + entry=0x8003C8C4 if args.variant=='jp' else int(json.loads(source('seamless-contract.json').read_text())['languages'][0]['code']['0x8003bafc']['address'],16) if eu2_profile else 0x8003BAFC) + out = (args.output or ROOT/('build-jp/final/generated' if args.variant=='jp' else 'build-eu2-seamless/generated' if args.variant=='eu2' else 'build-eu-seamless/generated')).resolve() + assert out.is_relative_to(ROOT) out.mkdir(parents=True, exist_ok=True) rows, streams, decoded = [], [], {} source_bytes = decoded_bytes = gam_count = oracle_count = 0 diff --git a/eu2/widescreen_analysis.py b/eu2/widescreen_analysis.py new file mode 100644 index 0000000..60c6dd9 --- /dev/null +++ b/eu2/widescreen_analysis.py @@ -0,0 +1,80 @@ +"""Read-only discovery of regional equivalents of the official EU1 hooks. + +Candidate discovery is not a runtime authorization. Full raw windows, changed +instructions and ambiguity are recorded for review before any hook is enabled. +""" +import argparse, difflib, hashlib, importlib.util, json, struct +from pathlib import Path + +BASE=0x80010000 +def words(data): return struct.unpack('<%dI'%(len(data)//4),data[:len(data)//4*4]) +def mask(w): + op=w>>26 + if op in (2,3): return w&0xfc000000 + if op in (1,4,5,6,7,8,9,10,11,12,13,14,15,32,33,34,35,36,37,38,40,41,42,43,46,50,58): return w&0xffff0000 + return w + +def main(): + p=argparse.ArgumentParser();p.add_argument('--reference-root',type=Path,required=True);a=p.parse_args() + root=a.reference_root.resolve() + spec=importlib.util.spec_from_file_location('official_ws',root/'eu/widescreen_profile.py') + m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) + reference=(root/'disc/EXE/MAIN0.EXE').read_bytes()[2048:];rw=words(reference) + sites=sorted(set(m.CODE.values())|set(m.FIELDS.values())|set(m.TABLES.values())|{0x8004f624,0x8004f628}) + targets={'eu2-en':root/'disc/eu2/EXE/MAIN0.EXE','eu2-es':root/'disc/eu2/EXE/MAIN5.EXE', + 'eu2-pt':root/'disc/eu2/EXE/MAIN6.EXE','eu2-sv':root/'disc/eu2/EXE/MAIN7.EXE', + 'eu2-da':root/'disc/eu2/EXE/MAIN8.EXE','jp':root/'disc/jp/SLPS_011.44'} + report={} + for name,path in targets.items(): + raw=path.read_bytes();body=raw[2048:];tw=words(body);normalized=[mask(w) for w in tw] + index={} + for i in range(len(tw)-8):index.setdefault(tuple(normalized[i:i+8]),[]).append(i) + rows=[] + for pc in sites: + ri=(pc-BASE)//4;found=[];used=0 + for radius in (16,8,4,2): + lo=ri-radius;hi=ri+radius;key=tuple(mask(w) for w in rw[lo:lo+8]) + candidates=[i+radius for i in index.get(key,[]) if i+2*radius<=len(tw) and normalized[i:i+2*radius]==[mask(w) for w in rw[lo:hi]]] + if len(candidates)==1:found=candidates;used=radius;break + row={'eu1':hex(pc),'candidates':[hex(BASE+i*4) for i in found],'window_bytes':used*8} + if found: + ti=found[0];b=body[(ti-used)*4:(ti+used)*4] + row.update(address=hex(BASE+ti*4),sha256=hashlib.sha256(b).hexdigest(), + changes=[{'relative':(i-used)*4,'eu1':hex(x),'target':hex(y)} for i,(x,y) in enumerate(zip(rw[ri-used:ri+used],tw[ti-used:ti+used])) if x!=y]) + rows.append(row) + if name.startswith('eu2-'): + # Resolve repetitive call sites by their enclosing independently + # matched boundaries, never by a whole-region address delta. + for row in rows: + if 'address' in row:continue + pc=int(row['eu1'],16) + left=max((r for r in rows if 'address' in r and int(r['eu1'],16)pc),key=lambda r:int(r['eu1'],16)) + lo=int(left['eu1'],16);hi=int(right['eu1'],16) + delta=int(left['address'],16)-lo + if int(right['address'],16)-hi!=delta:continue + if [mask(w) for w in rw[(lo-BASE)//4:(hi-BASE)//4]]!=normalized[(lo+delta-BASE)//4:(hi+delta-BASE)//4]:continue + target=pc+delta;at=target-BASE + row.update(address=hex(target),sha256=hashlib.sha256(body[at-32:at+32]).hexdigest(),window_bytes=64, + anchored_interval=[hex(lo),hex(hi)],changes=[{'relative':d,'eu1':hex(struct.unpack_from('pc),key=lambda r:int(r['eu1'],16)) + lo=int(left['eu1'],16);hi=int(right['eu1'],16);tl=int(left['address'],16);th=int(right['address'],16) + if not 0` cards. The official +memory-card picker functions have not been changed by the Japan integration. + +## Build requirements and Japan reproduction + +Windows x64, Python 3.11+, Clang/clang++, CMake and Ninja are required. Set +`TOMBI_TOOLCHAIN_BIN` to their bin directory if necessary. Recursively fetch the +pinned sources first; supply the pinned libwebp source tree or allow the normal +framework dependency fetch. OpenBIOS is supplied by the framework; select only +OpenBIOS, not a Sony BIOS. Original discs are private read-only inputs. + +```powershell +cmake -S psxrecomp/recompiler -B build-eu-tools -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ +cmake --build build-eu-tools --target psxrecomp-game psxrecomp-bios -j 3 +cmake -DOUT=psxrecomp/runtime/include/overlay_codegen_hash.h -DPSXRECOMP_CODEGEN_HASH_ROOT=psxrecomp -P psxrecomp/runtime/hash_codegen.cmake +python -I -B jp/build.py --enhancements --disc "path/to/Japan.cue" --webp-source "path/to/pinned/libwebp" +python -I -B jp/build_aot.py --config jp/pc.toml --all-areas --flavor 2 --work-dir build-jp/rc2/aot +python -I -B jp/build.py --enhancements --player --disc "path/to/Japan.cue" --webp-source "path/to/pinned/libwebp" +``` + +The scripts check full original-disc identity and original boot/overlay bytes. +They build separate private QA and production hosts; old prototype output paths +remain untouched. The production build has no debug server. Existing shared +runtime PGXP support is selected with the wrapper's PGXP keyword. The shared +Seamless, FMV and widescreen implementations use verified Japanese profiles in +the enhanced build; Seamless defaults on (like USA), FMV Skip and Widescreen off. +Japanese geometry HLE is not registered yet (beads-eio.4.35); Hybrid Controller +is not needed because the digital pad's stick fold moves Tomba. + +`jp/overlay-contracts.json` explains the 14 byte-checked data tails needed to +prevent native discovery from treating data as code. The remaining original +code/data and safe fallback are preserved. See `PORT_STATUS_EN.md` for exact +regional blockers and the limits of the evidence. 24 translated overlay files +and successful first-area play do not establish completion of the whole game. + +## European and USA backends / combined staging + +The previous EU build and test workflow remains `europe/build.py` and is +documented in `europe/DEVELOPER.md`. Reuse a pristine audited EU RC2 stage at +`dist/TombiRecomp-Europe-Windows-x64-RC2`; do not reuse a played installation. +Earlier isolated EU rebuild and all-ten-language results are retained rather +than repeated for unchanged executable bytes. + +For USA, use the existing upstream preparation/code-generation workflow with +`game.toml`, then configure the root project in `build-regional/us-player/native` +with Release, `PSX_DEBUG_TOOLS=OFF`, `PSX_PGXP_VARIANT=OFF`, +`PSXRECOMP_BIOS_STEMS=OpenBIOS`, `BUILD_TESTING=OFF`, and the same Clang toolchain. +Build target `psx-runtime`. Use `tools/build_us_aot.py` and `aot/overlays.json` +for original-disc native inputs. This candidate reuses 176 native pairs from +the official USA cache, validated against a newly extracted 25-recipe inventory +with `audit_aot_cache.py` and the current compiler/configuration. The stage +helper expects that private cache copy at `build-regional/us-cache-copy` and +the resulting receipt at `build-regional/us-audit-full/audit.json`. It verifies +artifact hashes again before copying. Legacy caches are not blindly trusted. +A fresh differently partitioned native cache can require adapting the expected +pair count in staging/audits; counts are packaging assertions, not a reason to +alter original bytes or suppress a failing producer. + +`python -I -B jp/stage_regional.py --enhancements --output dist/TombaRecomp-Regional-Japan-RC2` assembles a NEW four-region stage. It strips +debug paths only from new copied binaries, copies only required seed inputs, +and excludes user selections/cards, ROMs and developer logs. Keep all licenses +and the runtime overlay toolchain. These embedded fallback tools are needed +for portable execution; no system compiler/Python installation is needed. + +## Verification and isolated rebuild + +```powershell +python -I -B jp/acceptance.py --name fresh-default --opengl --build-dir build-jp/rc2/qa/native --config jp/pc.toml --seamless off +python -I -B jp/acceptance.py --name fresh-combined --opengl --build-dir build-jp/rc2/qa/native --config jp/pc.toml --pgxp --seamless on --fmv-skip +python -I -B jp/package_test.py --package "path/to/Windows.zip" --name fresh-package --all-regions +python -I -B jp/rebuild.py --enhancements --handoff "path/to/developer-folder" --name fresh-isolated +``` + +The acceptance tests use normal controller sequences, status reads and internal +framebuffer capture. There are no game-state writes or desktop actions. Each +uses new private cards. Install Unicorn 2.1.4 into a private test environment to +run `jp/test_codec.py` (638 files, two fill patterns, original Japanese decoder). +The HD installer fixture is shared with `eu/tests/test_hd_install.py`, using +`--game-id SLPS-01144` and the existing C++ test fixture binary. + +The isolated reconstruction applies all three patches to official source +archives, rebuilds compiler and Japanese player, regenerates boot C and extracts +original overlay recipes again. Reused native DLLs pass fresh ABI, pair, bounds +and original-byte guard audits. It does not claim that every DLL was recompiled +or that unchanged USA/EU hosts were rebuilt in that run. Reports distinguish +previous evidence, new automated checks and owner-confirmed prototype tests. + +Use `europe/audit.py --kind regional` for the player directory and ZIP, and +`--kind developer` for this handoff. Both archives must match their full staged +file manifests and pass independent private-path/card/disc scans. The owner +played and accepted the enhanced build on 2026-10-24; saving/loading on the +earlier prototype was already owner-confirmed. + +## RC2 regional implementation and targeted tests + +See RC2_MOD_REVIEW_EN.md. Generate profiles with `jp/seamless_profile.py` and +`jp/fmv_profile.py`; CMake runs both automatically in enhancement builds. +The existing EU loader, C++ resident preparation and FMV plugin remain shared. +The Japanese source generator pins the common loader hash, exact original +instructions, request descriptors, protected memory and callback contracts. +It removes the EU-only language/GP list exception and narrows skipped yields to +same-batch loading states. Script waits, movie streams, card code and clock stay +original. Failed preparation or a rejected request retains the original loader. + +Run the `tomba-jp-loader-tests` and `tomba-jp-prepare-tests` CMake targets in the +private QA build. The latter takes a copied JP CUE and private cache directory; +its optional fourth argument emits codec-state CSV for `jp/test_codec_state.py`. +`jp/test_fmv_teardown.py` checks 22 isolated original-MIPS teardown paths using +Unicorn 2.1.4. Offline fixtures are explicitly separate from controller-only +regular gameplay acceptance. `jp/card_probe.py` copies owner input cards, +exposes only pad inputs/read-only diagnostics and verifies original hashes. +Never distribute its private outputs or any memory cards. + +Old `jp/game.toml` and default build paths remain usable for reproducing RC1. +RC2 uses `jp/pc.toml`, four native filter entries and a distinct cache ABI tag. +Never mix RC1 and RC2 translation artifacts without current audits. diff --git a/jp/FEATURES_EN.md b/jp/FEATURES_EN.md new file mode 100644 index 0000000..cb2d9e2 --- /dev/null +++ b/jp/FEATURES_EN.md @@ -0,0 +1,31 @@ +# Regional feature scope + +| Feature | USA | EU1 | EU2 | Japan | +| --- | --- | --- | --- | --- | +| Common launcher and checked regional routing | Yes | Yes | Yes | Yes | +| Original game languages | English | Five original languages | Five original languages | Japanese | +| Separate settings and default cards | Yes | Yes | Yes | Yes | +| Native boot / guarded overlay fallback | Yes | Yes | Yes | Yes, 24 precompiled images | +| Normal controller, audio, Fast Forward | Shared implementation | Shared implementation | Shared implementation | Shared implementation; physical-device check remains manual | +| Seamless Loading | Existing | Existing | Existing | Shared loader with checked Japanese profile; default on, like USA | +| Geometry Optimization | Existing | Existing | Existing | Not yet: original MIPS path (same output, slower); Japanese packet ABI differs, tracked as beads-eio.4.35 | +| PGXP Precision | Existing framework support | Existing | Existing | Verified in ordinary gameplay; off by default | +| Hybrid Controller | Existing | Existing | Existing | Not needed: the original driver is digital-only, and the left stick folds onto the D-pad (stick and D-pad both move Tomba) | +| HD Texture Packs | External matching pack | External matching pack | External matching pack | Shared renderer; Japanese-target installer fixtures passed | +| Bezel Artwork | External image | External image | External image | Shared renderer; external image | +| FMV Skip | Existing, default off | Existing, default off | Hidden | 22 checked original movie bindings; default off | +| Widescreen renderer | Existing, default off | Existing, default off | Hidden | Checked shared-renderer adapter; default off | + +The unchanged EU backends retain their previous defaults and test results. +Japanese defaults preserve the prototype baseline (native scale, digital pad, +original SPU path, NTSC-J, windowed OpenGL, VSync off). Mod defaults match +USA: Seamless Loading on, the other optional mods off. Existing user settings +always take precedence. The launcher retains the official memory-card Browse +implementation. + +Owner-confirmed Japanese prototype results include boot, controls, Japanese +text/menus and saving/loading. On 2026-10-24 the owner played the enhanced build +(Widescreen and Seamless Loading on) and accepted it. Automated checks on current +psxrecomp master pass original New Game, walk/jump/land, left-stick movement, +audio, native overlay dispatch and 16:9 widescreen (`jp/acceptance.py +--stick-check --view 16:9`). Campaign-wide play remains open. diff --git a/jp/PORT_STATUS_EN.md b/jp/PORT_STATUS_EN.md new file mode 100644 index 0000000..da43b97 --- /dev/null +++ b/jp/PORT_STATUS_EN.md @@ -0,0 +1,78 @@ +# Japan port handoff status + +The original Japanese SLPS-01144 boot program and all 24 SYSTEM code images +are translated through the existing framework. Native overlay dispatch remains +guarded by the original bytes. Unsupported or invalidated code can use the +existing interpreter fallback; complete static execution coverage is not claimed. + +The initial prototype was manually confirmed to boot, display Japanese text, +accept controller input, and save and load. On 2026-10-24 the owner played the +enhanced build (Widescreen and Seamless Loading on; the checklist included the +first-house transition and return) and accepted it for merge. These +confirmations do not establish completion of the entire game. The original Japanese title menu includes English labels; +the Japanese in-game menus differ from the western releases by design. + +## Reviewed native boundaries + +`overlay-contracts.json` pins whole-file SHA-256 values and the final data +intervals described by byte-matching Japanese psx_tomba profiles, reference +revision fb3b36715f6966c099be95564c155bd842003b76. In these intervals the native +producer must not discover random data as instructions. Original byte guards +and the interpreter fallback remain active. In particular X02 previously +discovered unsupported words in its documented data tail. This is a producer +boundary correction, not a replacement or patch to the disc. + +## Deferred regional bindings + +Geometry Optimization: Japanese candidate routines differ substantially in +argument register use, packet lengths, and packet layout. Shared western HLE +functions are not an established match. Bindings need verified entry/end +addresses, original-byte signatures, and differential coverage of registers, +packet memory, return behavior, and unsupported cases. Until then the original +MIPS path runs (same output, slower); tracked as beads-eio.4.35. + +Seamless Loading RC2: the unchanged common EU batch loader is generated with +original-byte-checked Japanese bindings by `jp/seamless_profile.py`. The shared +catalog and C++ preparation path verify 1054 resources and 638 GAM files. Atomic +batch, VRAM/WVD, MAIN protection, hash rejection, original-loader fallback and +codec output/final-bit-state fixtures pass. Ordinary original New Game, Japanese +dialogs, walk/jump/land and audio pass with the loader active. No script timer, +original state or NTSC clock is rewritten. Streaming movies stay original. +After the owner's 2026-10-24 playtest with the loader active it defaults ON, +like the USA release. + +FMV Skip RC2: the shared western plugin is reused without code changes. All 22 +Japanese descriptor pointers, real STR sector headers and original frame totals +are checked; the Japanese table is 0x80071D3C and movie byte is 0x1F8001CD. +Original teardown/comparison code passes 22 isolated MIPS ON/OFF tests. Original +New Game also passes with the plugin active. These isolated tests do not prove +full-game event progression. Default OFF, like the USA release. + +Widescreen: `jp/widescreen_profile.py` binds the common renderer to the Japanese +MAIN (HUD callgraph, area 0/1 hooks, sprite tagging) against the reviewed +`jp/widescreen-contract.json`, and `jp/pc.toml` carries the Japanese culling +sites. It is part of the enhanced build (`--enhancements`) and defaults OFF, like +the USA release. Checked: 16:9 first-area acceptance (426x224 view, corner HUD) +and the owner's 2026-10-24 playtest; other areas are covered by ordinary play. + +Launcher localization is a separate project. This port preserves original game +resources and uses the existing English launcher and official memory-card picker. + +Hybrid Controller: Japanese vblank 0x80016994 calls PadRead at 0x80064B74, +which uses BIOS B0:0x16 and legacy buffer 0x8009DDB0. The western direct-pad +analog parsing contract is absent from this path. Reusing the western dynamic +pad presentation previously timed out before New Game. The game reads a digital +pad only, so the western analog/digital switch has nothing to switch: the host +folds the left stick onto the D-pad for a digital pad, and stick and D-pad both +move Tomba (`jp/acceptance.py --stick-check`). No Hybrid Controller mod is +offered or needed. See `RC2_MOD_REVIEW_EN.md` for evidence. + +## Manual acceptance + +Owner acceptance of the enhanced build: 2026-10-24. For later candidates: +use only a new card or a copy of an existing card. Load the Japanese save, cross +one ordinary area boundary, return, and check graphics, text, controls and audio. +Save normally, exit, restart using the common launcher, and load again. Report +both area names and any visual/audio or progression problems. The prototype's +save/load confirmation need not be repeated solely to restate the old result; +this final pass checks the expanded native overlay build. diff --git a/jp/RC2_MOD_REVIEW_EN.md b/jp/RC2_MOD_REVIEW_EN.md new file mode 100644 index 0000000..2048210 --- /dev/null +++ b/jp/RC2_MOD_REVIEW_EN.md @@ -0,0 +1,108 @@ +# Japan RC2: shared implementations and verified regional bindings + +Reference: byte-checked original SLPS-01144 boot SHA-1 +`07d577698ad31ec1a69b9d7b80a7c3f90b650888`, original disc SHA-256 +`fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b`. +The local psx_tomba reference is pinned at +`fb3b36715f6966c099be95564c155bd842003b76`; original instructions are decisive. + +| Feature | Reuse category | RC2 result | +| --- | --- | --- | +| Seamless Loading | Shared EU loader with a checked regional profile | Implemented, optional/default off pending regular transition acceptance | +| FMV Skip | Shared plugin unchanged, Japanese table/descriptor bindings | Implemented, optional/default off | +| PGXP | Existing framework implementation | Retained; combined ordinary-gameplay counters pass | +| Controller, audio, Fast Forward, cards | Shared runtime | Retained; no regional replacement | +| HD textures / bezel | Shared renderer and installer | Retained RC1 tests; matching external assets required | +| Geometry Optimization | Shared HLE ideas, different packet ABI | Hidden; larger adaptation and differential tests needed | +| Hybrid Controller | Shared policy, different original input driver | Hidden; adapter and original-menu/analog acceptance needed | +| Widescreen | Shared renderer, unverified Japanese camera/HUD contracts | Hidden; no new renderer work begun | + +Status since 2026-10-24 (this review records RC2 as delivered): the enhanced +build now also carries Widescreen, Seamless Loading defaults on like USA after +the owner's playtest, and Hybrid Controller is not needed because the digital +pad's stick fold already moves Tomba. Geometry Optimization is still open. See +`FEATURES_EN.md` and `PORT_STATUS_EN.md`. + +## Seamless profile + +`jp/seamless_profile.py` generates a regional derivative of the existing +`eu/seamless/loader.c`, not a second maintained loader. The generator pins its +source hash and checks original calls, instruction words and code hashes. + +| Role | Verified Japanese binding | +| --- | --- | +| openTask / sleepTask | 0x80016740 / 0x800167C0 | +| gameTask / sole root sleep continuation | 0x80019608 / 0x80019728 | +| cosmetic loading animation / render | 0x8001B808 / 0x80043A90 | +| original CD task | 0x8001F574 | +| GPU callback continuation / DrawSync | 0x8001FA48 / 0x8005BA48 | +| WVD continuations | 0x8001FB5C / 0x8001FC98 / 0x8001FCB8 | +| request queue / BCD file table | 0x800A0F70 / 0x800725F4 | +| transfer scratch / alternate arena | 0x800A5B28 / 0x800B58F8 | +| error bytes / game data | 0x8009F0F0 / 0x8009E508 | +| VAB header/body tables / bank table | 0x8009EF98 / 0x8009EE98 / 0x80072398 | +| decoder final bit/flags | 0x8009DBD0 / 0x8009DBD4 | +| protected resident MAIN end | 0x8009E000 | + +The original BCD table has 1047 slots, two unused. Japan has resident load +descriptors, so the western language-specific SYS/GP-list write exception is +removed. All MAIN protection, destination overlap, VRAM, WVD, code identity and +whole-batch preflight guards remain. Unsupported batches return to the original +CD task before partial installation. The diagnostic forced-CD mode is private +and demonstrates actual initial gameplay with zero installed resident batches. + +Only same-frame completed-batch loading yields may be skipped. Script waits, +opening sequences, ordinary gameplay yields, memory-card transfers, original +quests and NTSC timing are not modified. STR movies keep original streaming. +1054 resources are prepared on first activation; 638 GAM decoder results and +their final bit state were compared against original Japanese MIPS. + +New Game installs 13 first-area files plus a WSS batch; Japanese dialogs, +controller movement/jump/land and generated audio pass. A normal saved-game +area roundtrip remains manual: the bounded original-menu navigation did not +reliably reach the owner's save. No artificial state was used. OFF is retained +as the fresh-install default until this acceptance is complete. + +## FMV profile + +`jp/fmv_profile.py` validates all 22 original descriptor pointers at 0x80071D70, +their file-table identities and actual STR sector headers. Unlike the western +byte-index mapping, Japan uses pointers to 20-byte load records. The original +frame-total table at 0x80071D3C matches the common plugin's movie ordering; +movie ID is the original byte 0x1F8001CD. The original end comparison around +0x8001D454 and state-3/CdMix teardown are checked, not replaced with US addresses. +Some STR trailers intentionally extend beyond the original table's totals. + +The shared `src/mods/tomba_skip_fmv_plugin.c` is unchanged. Default OFF and the +existing Any-percent semantics are preserved. 22 isolated original-MIPS leaf +fixtures pass for ON/OFF behavior and table isolation; normal New Game also +passes with the active plugin. Later-film quest continuity is manual acceptance. + +## Concrete deferred contracts + +Geometry: Japanese neighborhoods emit packet lengths 7 at 0x80022B34 and 9 at +0x80022D04. Western HLE entry points 0x80024EEC, 0x800251C0 and 0x800253C8 +emit lengths 9, 9 and 12 and have different input/packet contracts. These are +observations, not approved Japanese entry mappings. Establish full Japanese +entry/end boundaries, argument/register effects, packet layout, returns and +unsupported-case equivalence before registering shared HLE implementations. + +Hybrid: Japanese vblank calls legacy PadRead at 0x80016994 -> 0x80064B74, +using BIOS B0:0x16 through 0x80064BBC and buffer 0x8009DDB0. The western +PadInitDirect/PadStartCom analog-report parser is not the same contract. +An actual driver adapter and original-title/menu/analog tests are needed. + +Widescreen: qualify Japanese projection/camera state, sprite transforms, +culling, HUD exclusions and geometry packet paths against the common renderer. +No complete small hook set is available; blindly transferring EU addresses +would be unsafe. Normal renderer and original Japanese layouts remain active. + +## Evidence and maintenance + +24 native pairs / 7277 range rows were regenerated for RC2's four entry filters, +then audited against original bytes and current ABI. Full static coverage is +not claimed; interpreter fallback is retained. EU/USA binaries are copied +unchanged from their previously audited pristine stages. No common launcher, +official Browse function, original ROM or real card is changed by this work. +See TEST_REPORT.json for current vs retained evidence and MANUAL_TESTS_DE.md +for the remaining owner acceptance. diff --git a/jp/README.md b/jp/README.md new file mode 100644 index 0000000..577261a --- /dev/null +++ b/jp/README.md @@ -0,0 +1,72 @@ +# Japan native port (SLPS-01144) + +Uses the existing framework and launcher, with no US/EU runtime source changes. +Original raw track: 289430064 bytes, SHA256 +`fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b`. +Boot SHA1: `07d577698ad31ec1a69b9d7b80a7c3f90b650888`. +Seeds are the pinned byte-matched Japan reference function addresses plus entry. +Boot loads at 0x80010000, entry 0x80065D80, GP initialized by original code to +0x8009DAC8. Overlay destination 0x800E9AF8 is checked at original loader words +0x8002025C/60. No US/PAL function hooks are registered. + +Use Windows Clang/CMake/Ninja on PATH (or TOMBI_TOOLCHAIN_BIN) and Python 3.11+. +The existing `build-eu-tools/psxrecomp-game.exe` must match the framework checkout. +The player uses the existing OpenBIOS source/backend workflow. + +``` +python -I -B jp/build.py --disc path/to/Japan.cue +python -I -B jp/build_aot.py --all-areas --flavor 2 --work-dir build-jp/full/aot-pgxp +python -I -B jp/build.py --player --disc path/to/Japan.cue +python -I -B jp/stage_regional.py +``` + +`build.py --webp-source ` accepts a pinned libwebp source tree; +the default reuses the existing local dependency cache. Build the shared +`psxrecomp-game` and `psxrecomp-bios` tools into `build-eu-tools` first and run +the canonical `psxrecomp/runtime/hash_codegen.cmake` hash-header generator. +All 24 Japanese SYSTEM code files now pass both native flavors' audits: +7,277 manifest rows per flavor. Reviewed, hash-checked data tails resolve the +earlier X02 discovery failure. This is not proof of complete static execution. +Runtime gaps retain the standard guarded native-cache/interpreter fallback. + +QA uses `jp/probe.py --name new-test-folder --hold 600`; the local debug endpoint +is only present in the private QA executable. Tests permit ordinary controller +input, RAM/status reads and internal framebuffer captures, never RAM writes, +state injection, forced inventory/quests or desktop automation. Each run creates +fresh isolated cards. Do not access a port belonging to another test process. + +Reproducible first-area acceptance (fresh normal New Game and controller inputs): +`python -I -B jp/acceptance.py --name new-opengl-test --opengl`. +The OpenGL test uses the existing hidden rendering context, not desktop actions. +Add `--pgxp` for the validated precision path. The default checks PGXP off. +`python -I -B jp/package_test.py --package --name --all-regions` +checks all four real launcher callbacks and production Japan/USA boot without +developer tools. `jp/test_codec.py` needs Unicorn 2.1.4 and compares 638 assets +against the Japanese decoder's original MIPS instructions, not another port. +`--view 16:9` (enhanced build) checks the widescreen renderer, `--stick-check` +checks left-stick movement through the host input layer, and `--try-roundtrip` +attempts the first-house transition (a bounded route; manual play is decisive). +Saving/loading was confirmed manually on the prototype; the owner played and +accepted the enhanced build on 2026-10-24. + +The new common package reuses both exact EU-RC2 executable files. The generic +regional catalog supports all four profiles; no launcher UI fork is needed. +Japan config/cards live under backends/SLPS-01144. Launcher localization remains +a separate task. `jp/rebuild.py --handoff --name ` +reconstructs the recorded upstream bases and patches, rebuilds the compiler and +Japanese player, then re-audits reused native pairs against freshly extracted +original bytes. See PORT_STATUS_EN.md for deferred regional bindings. +The staging script expects the independently verified EU RC2 base, new USA host +and original-disc-audited USA cache; it never uses a played installation. + +## Enhanced build (release) + +Use `jp/build.py --enhancements --disc ` and the corresponding `--player` +build, with `jp/build_aot.py --config jp/pc.toml --all-areas --flavor 2`. +It uses `jp/pc.toml`, `build-jp/rc2` and a separate audited native tag, and +carries Seamless Loading (default on, like USA), FMV Skip (default off) and +Widescreen (default off). The shared EU loader, FMV plugin and widescreen +renderer use checked Japanese profiles; see FEATURES_EN.md. +For the isolated rebuild add `--enhancements` to `jp/rebuild.py`. +The owner save before the hut was located and copied privately. Original hashes +are unchanged. diff --git a/jp/REGIONAL_README_EN.md b/jp/REGIONAL_README_EN.md new file mode 100644 index 0000000..109da45 --- /dev/null +++ b/jp/REGIONAL_README_EN.md @@ -0,0 +1,43 @@ +# TombaRecomp regional Windows package + +Extract the complete package to a new writable directory and run +`Start-TombaRecomp.cmd` (or `TombiRecompEU.exe`, the shared entry point). +Use the original launcher Browse control to select your legal CUE/BIN disc. +The launcher checks the serial, track size and complete SHA-256, then restarts +into the matching native backend. PLAY uses that backend's normal verification. +The last selected disc is remembered. No game files are included. + +| Disc | Original game languages | Backend directory | +| --- | --- | --- | +| SCUS-94236 | English | backends/SCUS-94236 | +| SCES-01330 Rev 1 | English, French, German, Italian, Dutch | package root | +| SCES-01331 | English, Spanish, Portuguese, Swedish, Danish | backends/SCES-01331 | +| SLPS-01144 | Japanese | backends/SLPS-01144 | + +Each backend keeps its own settings and `saves//card1.mcd` / `card2.mcd` +under its directory. Existing installations and cards are not imported or moved. +To test an existing save, select a COPY using the original memory-card picker. +Cross-region save compatibility is not established. Launcher language remains +English and is independent of the game's original languages. + +The US and European feature sets are retained. EU2 still hides unverified FMV +skip and widescreen. Japan provides normal graphics/audio, digital controller, +native boot and guarded overlays with interpreter fallback, and optional PGXP. +HD textures require a matching external pack; bezel artwork requires an image. +These remain off initially. Japan also offers Seamless Loading (on by default, +like USA), FMV Skip and Widescreen (both off by default) under Mods. The left +stick and the D-pad both move Tomba, so no Hybrid Controller mod is needed. +Japanese Geometry Optimization is not available yet; the original code runs. +Fast Forward and normal audio/video settings use the shared framework. + +Japan starts with native resolution, windowed OpenGL, VSync off, original digital +pad and original SPU path. PGXP is off; select at least 2x internal scale to see +sub-pixel rendering clearly. Existing settings take precedence over defaults. +Original NTSC-J timing and Japanese text resources are preserved. English words +on the original Japanese title menu are intentional. Confirm Japanese dialogs +with Circle; Cross jumps in gameplay. + +See `PORT_STATUS_EN.md` for verified scope and deferred bindings. A whole-game +playthrough is not claimed. OpenBIOS is included under its license; Sony BIOS and original game +resources are not included. No development tools need to be installed to play; +the bundled overlay fallback tools are runtime dependencies. diff --git a/jp/acceptance.py b/jp/acceptance.py new file mode 100644 index 0000000..3ce0758 --- /dev/null +++ b/jp/acceptance.py @@ -0,0 +1,171 @@ +"""Fresh original New Game, walk/jump/land; pad inputs and reads only.""" +import argparse,json,os,socket,struct,subprocess,sys,time +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'eu')) +from campaign_test import Probe,step,ALLOWED +ALLOWED.update(['quit','video_info','spu_status','spu_voices','geom_correction','mod_counters','resident_status','fmv_state','gpu_state','ws_tag_stats','ws_backdrop_ring','wide_shot','display_timing']) + +def main(): + p=argparse.ArgumentParser();p.add_argument('--name',required=True);p.add_argument('--opengl',action='store_true');p.add_argument('--port',type=int,default=4516);p.add_argument('--build-dir',type=Path,default=ROOT/'build-jp/full/qa/native');p.add_argument('--config',type=Path,default=ROOT/'jp/game.toml');p.add_argument('--pgxp',action='store_true');p.add_argument('--seamless',choices=['on','off','original']);p.add_argument('--try-roundtrip',action='store_true');p.add_argument('--fmv-skip',action='store_true');p.add_argument('--trace',action='store_true') + p.add_argument('--profile-defaults',action='store_true');p.add_argument('--require-hq',action='store_true');p.add_argument('--view',choices=['off','Fit','16:9','21:9','32:9'],default='off');p.add_argument('--audio-diagnostics',action='store_true');p.add_argument('--require-bgm',action='store_true');p.add_argument('--stick-check',action='store_true');a=p.parse_args() + build=a.build_dir.resolve();assert build.is_relative_to(ROOT/'build-jp'), 'Private Japan QA build required' + assert Path(a.name).name==a.name + out=ROOT/'build-jp'/a.name;out.mkdir(exist_ok=False);probe=Probe(out,a.port) + try: + with socket.create_connection(('127.0.0.1',a.port),timeout=.2):raise RuntimeError('Test port occupied') + except OSError:pass + env=dict(os.environ,PSX_OVERLAY_AUTOCOMPILE_OFF='1',PSX_INTERNAL_RESOLUTION='native',SDL_AUDIO_DRIVER='dummy',SDL_AUDIODRIVER='dummy') + if a.profile_defaults:env.pop('PSX_INTERNAL_RESOLUTION',None) + if a.trace:env['TOMBI_SEAMLESS_TRACE']='1' + if a.seamless=='original':env['TOMBA_JP_SEAMLESS_TEST_CD']='1' + if a.seamless:env['PSX_RESIDENT_CACHE']=str(ROOT/'build-jp/rc2/resident-cache') + def player(): + b=probe.ram(0x800A7B78,48);return dict(state=list(b[4:8]),position=[v/65536 for v in struct.unpack_from('=1500:break + except (OSError,RuntimeError):pass + if time.monotonic()>deadline:raise TimeoutError('boot') + time.sleep(.1) + for i in range(40): + s=player() + if s['state'][0] in [1,5] and s['position'][:2]==[64.0,-52.0] and probe.ram(0x8009E518,1)==b'\x04':break + probe.drive([step(['start'],16),step([],60),step(['cross'],16),step([],120)]) + else:raise TimeoutError('Original New Game') + snapshot('new-game') + for i in range(80): + probe.drive([step(['circle'],4),step([],35)]) + if player()['state'][0]==1: + probe.drive([step([],120)]) + if player()['state'][0]==1:break + else:raise TimeoutError('Original opening dialogs') + before=snapshot('walk-before');probe.drive([step(['right'],40),step([],4)]);after=snapshot('walk-after') + assert after['position'][0]>before['position'][0]+5 + probe.drive([step(['cross'],4),step([],8)]);jump=snapshot('jump-air');assert jump['state'][1]==2 and jump['position'][1]land['position'][0]+5,stick + audio_samples=[] + if a.audio_diagnostics or a.require_bgm: + # Quiet normal gameplay after transient jumping/landing effects expire. + # A nonzero mixer alone cannot distinguish BGM from sound effects. + for i in range(4): + probe.drive([step([],180)]) + status=probe.rpc('spu_status');voices=probe.rpc('spu_voices') + audio_samples.append(dict(frame=probe.rpc('frame')['frame'], + bgm_sequence_id=struct.unpack('=0 and s['bgm_vab_id']>=0 for s in audio_samples),'Original BGM sequence/bank did not open' + assert audio_samples[-1]['status']['nonzero_frames']>audio_samples[0]['status']['nonzero_frames'],'No sustained audio during quiet gameplay' + assert all(any(v['active'] and int(v['env'],16)>0 for v in s['voices']['voices']) for s in audio_samples),'No sustained SPU voices' + if a.require_hq: + assert audio_samples and all(s['timing']['spu_hq_enabled'] for s in audio_samples),'HQ SPU is not enabled in the runtime' + assert any(s['timing']['spu_hq_proven'] for s in audio_samples),'HQ SPU did not enter verified output during quiet gameplay' + assert all(s['timing']['vsync_requested']==0 for s in audio_samples),'Fresh VSync is not OFF' + if a.view!='off': + renderer=dict(gpu=probe.rpc('gpu_state'),tags=probe.rpc('ws_tag_stats'),backdrop=probe.rpc('ws_backdrop_ring')) + ws=renderer['gpu']['ws'] + assert ws['mode']==2 and ws['nw_extra']>0 and ws['x_margin']>0 and ws['game_mode'] and ws['view_anchor'],ws + assert renderer['tags']['hud_roles']['hit']>0,renderer['tags'] + (out/'renderer.json').write_text(json.dumps(renderer,indent=2)+'\n') + probe.rpc('wide_shot',path=str(out/'world-wide.png')) + roundtrip=None + if a.try_roundtrip: + origin=land['current_area_section'];samples=[];crossed=None;returned=None + # A villager may start an ordinary conversation on the way; finish it with + # the same Circle presses as the opening dialogs (bounded, no state writes). + def finish_dialog(label): + if player()['state'][0]!=5:return + probe.drive([step([],60)]);samples.append(snapshot(label+'-dialog')) + for i in range(60): + probe.drive([step(['circle'],4),step([],35)]) + if player()['state'][0]==1:break + for i in range(12): + probe.drive([step(['right','cross'],24),step(['right'],56),step([],20)]) + s=snapshot('boundary-out-'+str(i));samples.append(s) + if s['current_area_section']!=origin and s['state'][0]==1: + crossed=s;break + finish_dialog('boundary-out-'+str(i)) + # The ordinary first-house entrance needs Up at its door; do not force + # an area, quest, position or cutscene. This is one bounded route only. + if not crossed and 950<=player()['position'][0]<=1100: + for i in range(12): + finish_dialog('house-door-'+str(i)) + # Up in place first, then small steps right: the door is a narrow + # trigger next to where the walk stops. + probe.drive(([step(['right'],4),step([],10)] if i else [])+[step(['up'],30),step([],180)]) + s=snapshot('house-door-'+str(i));samples.append(s) + if s['current_area_section']!=origin: + crossed=s;break + if crossed: + for i in range(20): + if player()['state'][0]!=5:break + probe.drive([step(['circle'],4),step([],90)]) + probe.drive([step(['down'],30),step([],180)]) + s=snapshot('house-down-exit');samples.append(s) + if s['current_area_section']==origin and s['state'][0]==1:returned=s + for i in range(12): + if returned:break + probe.drive([step(['left','cross'],24),step(['left'],56),step([],20)]) + s=snapshot('boundary-back-'+str(i));samples.append(s) + if s['current_area_section']==origin and s['state'][0]==1: + returned=s;break + roundtrip=dict(origin=origin,crossed=crossed,returned=returned,samples=samples,passed=bool(crossed and returned),bounded_attempt=True) + (out/'roundtrip.json').write_text(json.dumps(roundtrip,indent=2)+'\n') + overlay=probe.rpc('overlay_loader_status');assert overlay['dispatch_native']>0 and overlay['registered']>0 + video=probe.rpc('video_info');audio=probe.rpc('spu_status');assert audio['nonzero_frames']>0 + if a.opengl:assert video['gl']==1 and video['backend']=='opengl' + precision=probe.rpc('geom_correction') + counters={item['name']:item['count'] for item in probe.rpc('mod_counters')['counters']} + if a.seamless=='on':assert counters.get('tomba.seamless.batches',0)>0,counters + if a.seamless in ['off','original']:assert not counters.get('tomba.seamless.batches',0),counters + if a.seamless=='original':assert counters.get('tomba.jp.seamless.test_original',0)>0,counters + if a.pgxp:assert precision['pgxp']['dataflow_hit']>0 and precision['pgxp']['tri_precise']>0 and precision['perspective_triangles']>0,precision + report=dict(result='passed',fresh_original_new_game=True,controller_only=True,artificial_game_state=False,japanese_dialog_framebuffer=True,walk_before=before['position'],walk_after=after['position'],jump=jump['position'],landed=land['position'],native_overlay_dispatches=overlay['dispatch_native'],native_functions_registered=overlay['registered'],nonzero_audio_frames=audio['nonzero_frames'],renderer=video['backend'],gl=video['gl'],desktop_automation=False,area_transition=False,save_load=False) + report.update(profile_defaults=a.profile_defaults,require_hq=a.require_hq,widescreen=a.view,pgxp_enabled=a.pgxp,precision=precision,seamless=a.seamless,mod_counters=counters,roundtrip=roundtrip,stick=stick,fmv_skip=a.fmv_skip) + if audio_samples:report['bgm_diagnostics']=audio_samples + if a.fmv_skip: + totals=struct.unpack('<22H',probe.ram(0x80071d3c,44));assert totals[1]==3,totals + report['fmv_original_new_game_teardown']=True;report['fmv_totals_after']=list(totals) + (out/'result.json').write_text(json.dumps(report,indent=2)+'\n');print(json.dumps({k:v for k,v in report.items() if k!='bgm_diagnostics'}),flush=True) + finally: + if child.poll() is None: + try:probe.rpc('quit');child.wait(timeout=10) + except Exception:child.terminate();child.wait(timeout=10) + (out/'events.json').write_text(json.dumps(probe.events,indent=2)+'\n') + if previous is None:mod_state.unlink(missing_ok=True) + else:mod_state.write_bytes(previous) +if __name__=='__main__':main() diff --git a/jp/build.py b/jp/build.py new file mode 100644 index 0000000..c3fdbdd --- /dev/null +++ b/jp/build.py @@ -0,0 +1,31 @@ +"""Build one Japan backend against the existing framework and launcher.""" +import argparse,os,subprocess,sys,tomllib +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'eu')) +from toolchain import executable + +def main(): + p=argparse.ArgumentParser();p.add_argument('--disc',type=Path,required=True);p.add_argument('--player',action='store_true');p.add_argument('--jobs',type=int,default=4);p.add_argument('--build-dir',type=Path);p.add_argument('--enhancements',action='store_true');p.add_argument('--webp-source',type=Path,default=ROOT/'build-eu2/native/_deps/psx_libwebp-src');a=p.parse_args() + profile=ROOT/('jp/pc.toml' if a.enhancements else 'jp/game.toml') + generated=(profile.parent/tomllib.loads(profile.read_text())['recompiler']['out_dir']).resolve() + build=(a.build_dir or ROOT/'build-jp'/('rc2' if a.enhancements else 'full')/('player' if a.player else 'qa')/'native').resolve();assert build.is_relative_to(ROOT);build.parent.mkdir(parents=True,exist_ok=True) + env=dict(os.environ) + for key in ['TEMP','TMP']: + folder=build.parent/key;folder.mkdir(exist_ok=True);env[key]=str(folder) + mapping='-ffile-prefix-map='+ROOT.as_posix()+'=. -fmacro-prefix-map='+ROOT.as_posix()+'=.' + cmake=executable('cmake.exe') + commands=[('prepare',[sys.executable,'-I','-B','jp/prepare.py','--disc',str(a.disc.resolve())]), + ('generate',[str(ROOT/'build-eu-tools/psxrecomp-game.exe'),'--config',str(profile)]), + ('configure',[cmake,'-S','jp','-B',str(build),'-G','Ninja','-DCMAKE_BUILD_TYPE=Release', + '-DCMAKE_C_COMPILER='+executable('clang.exe'),'-DCMAKE_CXX_COMPILER='+executable('clang++.exe'),'-DCMAKE_MAKE_PROGRAM='+executable('ninja.exe'), + '-DCMAKE_C_COMPILER_LAUNCHER=','-DCMAKE_CXX_COMPILER_LAUNCHER=','-DCMAKE_C_FLAGS='+mapping,'-DCMAKE_CXX_FLAGS='+mapping, + '-DPSX_DEBUG_TOOLS='+('OFF' if a.player else 'ON'),'-DPSX_PGXP_VARIANT=OFF','-DPSXRECOMP_BIOS_STEMS=OpenBIOS','-DBUILD_TESTING=OFF', + '-DTOMBA_JP_ENHANCEMENTS='+('ON' if a.enhancements else 'OFF'),'-DTOMBA_JP_GENERATED_DIR='+str(generated), + '-DPython3_EXECUTABLE='+sys.executable,'-DFETCHCONTENT_SOURCE_DIR_PSX_LIBWEBP='+str(a.webp_source.resolve())]), + ('build',[cmake,'--build',str(build),'--target','tomba-jp','--parallel',str(a.jobs)])] + for label,cmd in commands: + print(label,flush=True) + with (build.parent/(label+'.log')).open('w') as log:subprocess.run(cmd,cwd=ROOT,env=env,stdout=log,stderr=subprocess.STDOUT,check=True) + print(build/'TombaRecompJP.exe',flush=True) +if __name__=='__main__':main() diff --git a/jp/build_aot.py b/jp/build_aot.py new file mode 100644 index 0000000..f3f31d7 --- /dev/null +++ b/jp/build_aot.py @@ -0,0 +1,52 @@ +"""Japan native overlays through the existing guarded AOT pipeline.""" +import argparse,hashlib,json,os,shutil,struct,sys,tempfile +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'eu'),str(ROOT/'psxrecomp/tools')] +from toolchain import executable +import aot_overlay_pipeline as pipeline +BASE=0x800E9AF8 + +def main(): + p=argparse.ArgumentParser();p.add_argument('--workers',type=int,default=2);p.add_argument('--destination',type=Path,default=ROOT/'build-jp/full/qa/native');p.add_argument('--prepare-only',action='store_true');p.add_argument('--all-areas',action='store_true');p.add_argument('--reuse-cache',type=Path);p.add_argument('--flavor',type=int,choices=[0,2],default=2);p.add_argument('--work-dir',type=Path);p.add_argument('--config',type=Path,default=ROOT/'jp/game.toml');a=p.parse_args() + work=(a.work_dir or ROOT/'build-jp/full/aot').resolve();assert work.is_relative_to(ROOT);work.mkdir(parents=True,exist_ok=True);tmp=work/'temp';tmp.mkdir(exist_ok=True);os.environ['TEMP']=os.environ['TMP']=str(tmp);tempfile.tempdir=str(tmp) + disc=pipeline.Disc(ROOT/'disc/jp/disc.cue') + profile=dict(schema='psxrecomp AOT methods v1',game_id='SLPS-01144',disc_hashes={'sha256':'fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b'},strict_bounds=True,bios_resident=False,images=[],checks=[]) + pipeline.check_disc(profile,disc,ROOT/'disc/jp/disc.cue') + boot=disc.read('SLPS_011.44');assert hashlib.sha1(boot).hexdigest()=='07d577698ad31ec1a69b9d7b80a7c3f90b650888' + # Exact original type-F0 destination materialization, also confirmed by the + # byte-matched Japan decompilation profiles. Never reuse a US/EU address. + pc=0x8002025C;values={hex(pc):'0x3c06800f',hex(pc+4):'0x24c69af8'} + for address,value in values.items():assert struct.unpack_from('= 1500: break + except (OSError, RuntimeError): pass + if time.monotonic()>deadline: raise TimeoutError('Boot') + time.sleep(.1) + sample('boot'); print('READY '+str(out), flush=True) + if a.load: + title_deadline=time.monotonic()+100 + while struct.unpack('<4H', probe.ram(0x801FD848,8))[:2] != (4,3): + if time.monotonic()>title_deadline: raise TimeoutError('Original title') + probe.drive([step(['start'],4),step([],60)]) + probe.drive([step(['down'],16),step([],35),step(['start'],16),step([],180)]) + sample('load-menu') + deadline = time.monotonic()+a.seconds + number = 1 + while time.monotonic()title_deadline: raise TimeoutError('Original title') + time.sleep(.1) + probe.drive([step(['down'],16),step([],35),step(['start'],16),step([],180)]) + if 'drive' in req: probe.drive(req['drive']) + result = sample(req.get('sample', 'step-%03d' % number)) + print(json.dumps(result), flush=True); number += 1 + finally: + if child is not None and child.poll() is None: + try: probe.rpc('quit'); child.wait(timeout=10) + except Exception: child.terminate(); child.wait(timeout=10) + if previous is None: state.unlink(missing_ok=True) + else: state.write_bytes(previous) + (out/'events.json').write_text(json.dumps(probe.events, indent=2)) + unchanged = all(hashlib.sha256((source/n).read_bytes()).hexdigest()==h for n,h in originals.items()) + (out/'originals-unchanged.json').write_text(json.dumps(dict(unchanged=unchanged))) + assert unchanged, 'Original cards changed' + +if __name__ == '__main__': main() diff --git a/jp/fmv_profile.py b/jp/fmv_profile.py new file mode 100644 index 0000000..53301e0 --- /dev/null +++ b/jp/fmv_profile.py @@ -0,0 +1,34 @@ +"""Verify Japanese FMV totals, descriptor indirection and original teardown.""" +import argparse,hashlib,json,mmap,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'psxrecomp/tools')) +from aot_overlay_pipeline import Disc,check_disc + +def generate(output=None): + disc=Disc(ROOT/'disc/jp/disc.cue');check_disc(dict(disc_hashes={'sha256':'fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b'}),disc,ROOT/'disc/jp/disc.cue') + exe=disc.read('SLPS_011.44');assert hashlib.sha1(exe).hexdigest()=='07d577698ad31ec1a69b9d7b80a7c3f90b650888';body=exe[2048:] + def word(pc):return struct.unpack_from('>4)*10+(v&15);rows=[] + with (ROOT/'disc/jp/disc.bin').open('rb') as f,mmap.mmap(f.fileno(),0,access=mmap.ACCESS_READ) as track: + for mid,total in enumerate(totals): + pointer=struct.unpack_from('=400:break + except (OSError,RuntimeError):pass + if time.monotonic()>deadline:raise TimeoutError('Original boot frame deadline') + time.sleep(.1) + sample('boot');print('READY '+str(out),flush=True) + deadline=time.monotonic()+a.hold + while a.hold and time.monotonic()>26==15 and lo>>16==opcode and (hi>>16&31)==(lo>>21&31),(hex(pc),hex(hi),hex(lo)) + return ((hi&65535)<<16)+(lo&65535)-(65536 if lo&32768 else 0) + row=dict(queue=field(0x8001f650,0x2442),files=field(0x8001f680,0x24a5),scratch=field(0x8001f764,0x2442),alternate=field(0x8001f754,0x2442),errors=field(0x8001f5b0,0xac20),headers=field(0x8001fb78,0x2484),bodies=field(0x8001fbb0,0x2484),codec_bit=field(0x8003c924,0xa020),codec_flags=field(0x8003c93c,0xa423),spu_mode=field(0x80070e28,0x8c63),spu_address=0x8009d780,game=0x8009e508,list_buffer=0,list_lba=0,list_size=0) + assert word(0x80070e2c)==0x3c01800a and word(0x80070e30)==0xa422d780 + assert word(0x8001fbf4)==0x3c018007 and word(0x8001fbfc)==0x8c342398 + # Unlike EU, Japanese WVDs put the VAB-header table FIRST and ADPCM + # bodies SECOND. The original loader derives s4/s2 from word 0, fp/s7 + # from word 4, then uses those bases for headers/bodies respectively. + sound_layout_words={0x8001fae4:0x8c430000,0x8001faec:0x0043a021, + 0x8001faf0:0x8c430004,0x8001faf4:0x02809021,0x8001faf8:0x0043f021, + 0x8001fafc:0x03c0b821,0x8001fb08:0x8e430000,0x8001fb10:0x00031882, + 0x8001fb14:0x3050000f,0x8001fb90:0x02821021,0x8001fbc8:0x03c21021} + for pc,value in sound_layout_words.items():assert word(pc)==value,hex(pc) + # Original loader's exact leaf calls and gameTask sleep continuation. + calls={0x80019720:0x800167c0,0x8001fa40:0x8005bd0c,0x8001fb54:0x8006e148,0x8001fc90:0x8006e29c,0x8001fcb0:0x8006e1cc} + for pc,target in calls.items():assert word(pc)==(0x0c000000|((target&0xfffffff)>>2)),hex(pc) + assert word(0x80019724)==0x24040001 and word(0x80019728)==0x080065ab + for pc in [0x8001ffe4,0x800200a4,0x8002077c]: + assert word(pc-8)==0x3c058002 and word(pc-4)==0x24a5f574 + assert word(pc)==0x0c0059d0 and word(pc+4)==0x24040002 + # BCD table: 1047 slots, including two unused zero entries. STR lengths + # count 2336-byte sectors; movies remain on the original streaming path. + by_lba={l:(name,n) for name,(l,n) in disc.files.items()};bcd=lambda v:(v>>4)*10+(v&15) + files=[] + for i in range(1047): + at=row['files']-0x80010000+i*8;raw=body[at:at+8] + if i in [217,218]:assert raw==bytes(8);continue + lba=(bcd(raw[0])*60+bcd(raw[1]))*75+bcd(raw[2])-150;name,size=by_lba[lba] + expected=size//2048*2336 if name.endswith('.STR') else size + assert struct.unpack_from('= 1119','id >= 1047').replace('p->files,1119*8','p->files,1047*8') + source=source.replace('0x8004EB08u,0x850','0x8004EB08u,0x8b0').replace('0x800198B4u,0x3154','0x800198B4u,0x2cd0') + # Preserve the shared planner/transfer implementation; adapt only the + # original regional WVD layout and handle-slot semantics. No waits/restarts. + replacements={ + 'uint32_t samples=u32(p), headers=u32(p+4);':'uint32_t samples=u32(p+4), headers=u32(p);', + 'uint32_t samples=r->dest+r32(r->dest), headers=r->dest+r32(r->dest+4);': + 'uint32_t samples=r->dest+r32(r->dest+4), headers=r->dest+r32(r->dest);'} + for old,new in replacements.items():assert source.count(old)==1;source=source.replace(old,new) + assert source.count('if(slot==15) slot=0;')==2 + source=source.replace('if(slot==15) slot=0;','/* Japanese original retains slot 15. */') + # Japan has resident descriptors, no language-specific GP/list DMA window. + begin=source.index(' unsigned id=r16(r->desc);');end=source.index(' if((a<',begin) + source=source[:begin]+' const int load_list=0; /* No Japanese SYS/LD lists. */\n'+source[end:] + old='return (state==0 && phase<=3) || (state==1 &&\n (phase==0 || phase==7 || (phase>=1 && phase<=6 && phase!=3 && step==0)));' + assert old in source + source=source.replace(old,'return last_batch_frame==frame && state==1 &&\n (phase==7 || (phase>=1 && phase<=6 && phase!=3 && step==0));') + source=source.replace('static int install_batch(CPUState *cpu) {','static int install_batch(CPUState *cpu) {\n if(diagnostic_cd_lists) { ++fallbacks; psx_mod_counter_add("tomba.jp.seamless.test_original",1); return 0; }') + source=source.replace('TOMBI_SEAMLESS_TEST_CD_LISTS','TOMBA_JP_SEAMLESS_TEST_CD').replace('EU seamless:','JP seamless:') + # Original script timers/intro waits and gameplay yields stay original. + def remap(m): + value=int(m.group(1),16) + if value in MAP:return hex(MAP[value])+'u' + if 0x10000<=value<0x78000 and value+0x80000000 in MAP:return hex(MAP[value+0x80000000]&0x1fffffff)+'u' + assert not (0x80010001<=value<0x80099001),hex(value) + return m.group() + source=re.sub(r'0x([0-9a-fA-F]+)u\b',remap,source) + (out/'jp_loader.c').write_text(source) + # Reuse the shared atomic-batch/sound/VRAM safety fixture. Replace only its + # EU-specific list-window checks and regional constants; no second planner. + test=(ROOT/'eu/seamless/tests.c').read_text().replace('"loader.c"','"jp_loader.c"').replace('lang<5','lang<1') + # The common synthetic guard fixture uses the EU table order; model the + # Japanese original order here as well. Runtime banks remain unmodified. + test=test.replace('{{0,0x100},{4,0x40}', '{{0,0x40},{4,0x100}') + start=test.index(' /* Only the original file-0 system-list extent');end=test.index(' w32(0x1f8001d4',start) + test=test[:start]+''' /* Japan has no list-write exception below MAIN's end. */ + w32(s_profile->queue+126*8+4,0x8009dac8);writes=0; + assert(!install_batch(&cpu) && !writes && !leaves); + setup(lang);diagnostic_cd_lists=1;writes=0; + assert(!install_batch(&cpu) && !writes && !leaves); + diagnostic_cd_lists=0;setup(lang);frame=123;last_batch_frame=frame; +'''+test[end:] + test=test.replace('cpu.gpr[31]=0x800199e8;cpu.gpr[4]=1;','cpu.gpr[31]=0x800199e8;cpu.gpr[4]=1;\n assert(!loading_yield_allowed(&cpu,1,0,0,0));\n assert(!loading_yield_allowed(&cpu,1,1,0,3)); /* scripted intro wait */\n last_batch_frame=frame-1;assert(!loading_yield_allowed(&cpu,1,1,7,0));last_batch_frame=frame;') + for old,new in MAP.items():test=test.replace(hex(old),hex(new)) + test=test.replace('int main(void) {','int main(int argc,char **argv) {') + pos=test.index(' puts(') + test=test[:pos]+''' assert(argc==2); + FILE *original=fopen(argv[1],"rb");assert(original); + assert(!fseek(original,2048,SEEK_SET)); + assert(fread(memory+0x10000,1,0x8e000,original)==0x8e000);fclose(original); + assert(validate_code());memory[0x16740]^=1;assert(!validate_code()); + puts("PASS Japan: shared atomic batches, WVD/VRAM guards, original-byte rejection, no GP exception, batch-local loading yields and original script waits"); + return 0; +} +''' + (out/'jp_loader_tests.c').write_text(test) + contract=dict(boot_sha1=hashlib.sha1(exe).hexdigest(),shared_loader_sha256=hashlib.sha256((ROOT/'eu/seamless/loader.c').read_text().encode()).hexdigest(),fields={k:hex(v) for k,v in row.items() if k!='fingerprint'},code_map={hex(k):hex(v) for k,v in MAP.items()},guard_spans=[dict(role=k,address=hex(pc),bytes=n,sha256=hashlib.sha256(body[pc-0x80010000:pc-0x80010000+n]).hexdigest()) for k,pc,n in spans],files=files,no_gp_list_exception=True,only_batch_local_loading_yields=True,wvd_header_table_word=0,wvd_body_table_word=1,wvd_slot_15_preserved=True) + (out/'loader-contract.json').write_text(json.dumps(contract,indent=2)+'\n') + print('PASS Japan loader bindings: 1047 slots, original calls/fields, no NTSC/script changes',flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--output',type=Path);generate(p.parse_args().output) diff --git a/jp/seeds/boot.txt b/jp/seeds/boot.txt new file mode 100644 index 0000000..17a20c9 --- /dev/null +++ b/jp/seeds/boot.txt @@ -0,0 +1,729 @@ +# Japan original boot SHA1 07d577698ad31ec1a69b9d7b80a7c3f90b650888 +0x80015d1c +0x80016548 +0x800165c4 +0x80016610 +0x800166e4 +0x80016740 +0x800167c0 +0x800167f4 +0x80016844 +0x800168b0 +0x80016908 +0x80016934 +0x80016960 +0x80016a20 +0x80016a88 +0x80016bd0 +0x8001798c +0x80017a1c +0x80017aac +0x80017d18 +0x80017db8 +0x8001c78c +0x8001e2d4 +0x8001e388 +0x8001e528 +0x8001ff98 +0x800208e4 +0x80020958 +0x80020c5c +0x80020d24 +0x80020d54 +0x80020de4 +0x80020e0c +0x80020e38 +0x8002122c +0x80021550 +0x80021704 +0x80022098 +0x800220f0 +0x8002222c +0x80022468 +0x800224c0 +0x800224f4 +0x800228ac +0x800228f0 +0x80022940 +0x800229d8 +0x80022ae8 +0x80022af0 +0x80022af8 +0x80027a5c +0x80027d58 +0x80028398 +0x8002d150 +0x8002e42c +0x8002edb0 +0x80030440 +0x80030620 +0x80030a68 +0x80030cdc +0x80030d78 +0x80030f30 +0x8003109c +0x8003112c +0x800311b0 +0x8003157c +0x80031584 +0x800315cc +0x8003229c +0x800323a8 +0x800324b4 +0x800325c0 +0x80034588 +0x80036044 +0x800360c0 +0x8003621c +0x80036350 +0x80036544 +0x80037b04 +0x80037d08 +0x80037d64 +0x80037e90 +0x80037f9c +0x80037fac +0x80038240 +0x80038a6c +0x80038bac +0x80038c04 +0x80038c60 +0x80038da8 +0x80038e10 +0x80038e38 +0x80038e70 +0x80038ea8 +0x80038f0c +0x80038f3c +0x80038f70 +0x80039024 +0x8003907c +0x800390e8 +0x800391f8 +0x80039300 +0x80039398 +0x800393f8 +0x8003942c +0x80039488 +0x800394e0 +0x800395cc +0x800396c0 +0x800398b0 +0x800398f0 +0x80039abc +0x80039f3c +0x80039f9c +0x8003a058 +0x8003a124 +0x8003a36c +0x8003a7b0 +0x8003aa7c +0x8003ab48 +0x8003aba4 +0x8003abf4 +0x8003ac40 +0x8003af90 +0x8003afe0 +0x8003b058 +0x8003b0c4 +0x8003b114 +0x8003b164 +0x8003b1bc +0x8003b20c +0x8003b284 +0x8003b2b8 +0x8003b2e8 +0x8003b35c +0x8003b38c +0x8003b3c0 +0x8003b434 +0x8003b49c +0x8003b508 +0x8003b5d0 +0x8003b688 +0x8003b804 +0x8003b830 +0x8003b884 +0x8003b8b0 +0x8003b8fc +0x8003b940 +0x8003b964 +0x8003b988 +0x8003ba18 +0x8003ba5c +0x8003baa0 +0x8003bb1c +0x8003bb6c +0x8003bbbc +0x8003bc0c +0x8003bc5c +0x8003bc80 +0x8003bca4 +0x8003bd38 +0x8003bd5c +0x8003bd7c +0x8003c8c4 +0x8003ca50 +0x8003ca98 +0x8003cb48 +0x8003fdd0 +0x80040b10 +0x80040e40 +0x80040eb4 +0x80041224 +0x800412b0 +0x800414a0 +0x80041514 +0x80041688 +0x800417c4 +0x800418f8 +0x80041a70 +0x80041b40 +0x80041d94 +0x80041e08 +0x80042048 +0x80042280 +0x80042870 +0x800428e8 +0x80042a84 +0x80042ce4 +0x80042ef4 +0x80043024 +0x80043728 +0x8004792c +0x8004b238 +0x8004b318 +0x8004b5a4 +0x8004b848 +0x8004b9f0 +0x8004ba68 +0x8004bb8c +0x8004bd10 +0x8004befc +0x8004c0f0 +0x8004c2f0 +0x8004c518 +0x8004c7bc +0x8004c83c +0x8004c94c +0x8004ca80 +0x8004cb14 +0x8004cf38 +0x80052f24 +0x80052fec +0x80053184 +0x8005339c +0x800535b0 +0x800537fc +0x8005391c +0x80053a88 +0x80053cd4 +0x80053d9c +0x80053dec +0x80053e3c +0x80053f18 +0x800540ac +0x8005415c +0x800542a0 +0x800545d4 +0x8005490c +0x800549a4 +0x8005845c +0x8005846c +0x8005847c +0x8005848c +0x8005849c +0x800584ac +0x800584bc +0x800584cc +0x800584dc +0x8005851c +0x8005857c +0x8005858c +0x800586cc +0x80058704 +0x80058738 +0x80058768 +0x8005879c +0x800587cc +0x80058874 +0x800588b8 +0x800588e8 +0x8005895c +0x800589d4 +0x80058a48 +0x80058a84 +0x8005910c +0x8005913c +0x8005916c +0x800591bc +0x80059270 +0x80059acc +0x80059bb0 +0x80059c04 +0x80059c3c +0x80059c6c +0x80059c7c +0x80059c8c +0x80059c9c +0x80059d04 +0x80059dbc +0x80059e4c +0x80059e5c +0x80059edc +0x80059f14 +0x80059fa0 +0x8005a03c +0x8005a048 +0x8005a0c4 +0x8005a0e4 +0x8005a120 +0x8005a15c +0x8005a180 +0x8005a1a4 +0x8005a2a0 +0x8005a334 +0x8005a3c4 +0x8005a45c +0x8005a4f4 +0x8005a63c +0x8005a640 +0x8005a670 +0x8005a9d4 +0x8005aac0 +0x8005ab28 +0x8005ab90 +0x8005ac60 +0x8005ac9c +0x8005ad6c +0x8005ad84 +0x8005ae54 +0x8005ae94 +0x8005aeb0 +0x8005aecc +0x8005af08 +0x8005af44 +0x8005af68 +0x8005af80 +0x8005afa8 +0x8005afd0 +0x8005afe4 +0x8005aff8 +0x8005b00c +0x8005b020 +0x8005b034 +0x8005b048 +0x8005b05c +0x8005b070 +0x8005b084 +0x8005b098 +0x8005b0ac +0x8005b0c0 +0x8005b0d4 +0x8005b0e8 +0x8005b0fc +0x8005b110 +0x8005b124 +0x8005b144 +0x8005b164 +0x8005b184 +0x8005b1a4 +0x8005b258 +0x8005b2b8 +0x8005b324 +0x8005b35c +0x8005b4dc +0x8005b588 +0x8005b70c +0x8005b820 +0x8005b884 +0x8005b930 +0x8005b940 +0x8005b950 +0x8005b9ac +0x8005ba48 +0x8005bab4 +0x8005bbdc +0x8005bc70 +0x8005bd0c +0x8005bd70 +0x8005bdd4 +0x8005be98 +0x8005bf50 +0x8005bfe8 +0x8005c048 +0x8005c0bc +0x8005c1c0 +0x8005c2dc +0x8005c314 +0x8005c7bc +0x8005c7f4 +0x8005c824 +0x8005c860 +0x8005c8e4 +0x8005c928 +0x8005c954 +0x8005c9ac +0x8005cbc4 +0x8005ce54 +0x8005ceac +0x8005cf78 +0x8005d044 +0x8005d088 +0x8005d10c +0x8005d1bc +0x8005d1d4 +0x8005d2bc +0x8005d518 +0x8005d754 +0x8005d9d8 +0x8005da00 +0x8005da14 +0x8005da64 +0x8005dab0 +0x8005dae0 +0x8005db04 +0x8005dde4 +0x8005e0d0 +0x8005e22c +0x8005e374 +0x8005e3a8 +0x8005e514 +0x8005e5f0 +0x8005e61c +0x8005e62c +0x8005e63c +0x8005e6a4 +0x8005e6e4 +0x8005e990 +0x8005eab0 +0x8005ec48 +0x8005fdc8 +0x8005fe04 +0x8005fe94 +0x8005ff34 +0x8005ff98 +0x800600ec +0x80060188 +0x80060190 +0x80060218 +0x800602a8 +0x800603b8 +0x80060518 +0x80060548 +0x80060598 +0x800605f8 +0x80060728 +0x80060758 +0x80060788 +0x800607b8 +0x800607d8 +0x800607e8 +0x800607f8 +0x80060808 +0x80060828 +0x8006083c +0x80060850 +0x8006085c +0x80060870 +0x80060888 +0x8006089c +0x800608b0 +0x800608c4 +0x800608d0 +0x800608dc +0x800608e8 +0x80060908 +0x80060928 +0x80060948 +0x80060958 +0x80060980 +0x800609a8 +0x800609e4 +0x80060a08 +0x80060a30 +0x80060a58 +0x80060a78 +0x80060a9c +0x80060af4 +0x80060b4c +0x80060b68 +0x80060b98 +0x80060e28 +0x80060fc8 +0x80061168 +0x80061308 +0x80061488 +0x800614f8 +0x80061528 +0x80061558 +0x800615e8 +0x80061610 +0x80061638 +0x80061660 +0x80061670 +0x80061680 +0x80061690 +0x800616a0 +0x800616b0 +0x8006171c +0x8006173c +0x80061754 +0x80061788 +0x800617bc +0x800617dc +0x800617fc +0x80061814 +0x8006182c +0x80061964 +0x80061a90 +0x80061bd4 +0x80061bf4 +0x80061c14 +0x80061c38 +0x80061c58 +0x80061d5c +0x80061ddc +0x80062364 +0x800625e4 +0x800628b0 +0x80062ccc +0x80062d54 +0x80062e34 +0x80062f28 +0x80062f78 +0x80063164 +0x800632d0 +0x800633d0 +0x800633e0 +0x800634c4 +0x80063514 +0x80063728 +0x800638f4 +0x80063944 +0x80063a24 +0x80063af0 +0x80063b08 +0x80063b9c +0x80063bbc +0x80063c1c +0x80063c74 +0x80063d04 +0x80063d64 +0x80063dec +0x80063ea4 +0x80063ee0 +0x80063fa4 +0x80063fc4 +0x80064934 +0x80064968 +0x80064fb4 +0x800650fc +0x80065198 +0x800651a8 +0x800651d8 +0x80065208 +0x80065238 +0x8006526c +0x8006529c +0x800652cc +0x800652fc +0x8006530c +0x80065324 +0x80065340 +0x8006541c +0x80065604 +0x80065758 +0x80065804 +0x800658b8 +0x800658c0 +0x800658d8 +0x800658e8 +0x800658f8 +0x80065908 +0x80065944 +0x80065988 +0x800659e0 +0x80065a58 +0x80065a84 +0x80065ab0 +0x80065b00 +0x80065c84 +0x80065d2c +0x80065d58 +0x80065d70 +0x80065d80 +0x80065e28 +0x80065e98 +0x80065f00 +0x80065f10 +0x8006608c +0x800660b0 +0x800660d4 +0x80066198 +0x80066280 +0x800662b0 +0x800665c0 +0x80066640 +0x80066a70 +0x80066b48 +0x80066c18 +0x80066cc8 +0x80066d58 +0x80066e90 +0x80066fcc +0x80067044 +0x800670bc +0x80067180 +0x80067210 +0x800672cc +0x8006735c +0x800673ec +0x800674a0 +0x800674fc +0x80067588 +0x80067640 +0x800676f0 +0x800677a0 +0x80067854 +0x8006790c +0x800679c0 +0x80067a68 +0x80067b44 +0x80067be4 +0x80067c84 +0x80067ca8 +0x80067cd0 +0x80067cf4 +0x80067d18 +0x80067d3c +0x80067df4 +0x80068034 +0x8006820c +0x800682f4 +0x8006836c +0x80068424 +0x8006873c +0x80068774 +0x800687ac +0x800687cc +0x800688cc +0x8006891c +0x80068b84 +0x80068bc4 +0x80068c04 +0x80068c54 +0x80068ec8 +0x800691a8 +0x80069448 +0x800694ec +0x80069518 +0x80069628 +0x800697ec +0x80069ad0 +0x80069bd8 +0x80069c40 +0x80069da8 +0x80069dd0 +0x80069dfc +0x80069fdc +0x8006a148 +0x8006a180 +0x8006a1b0 +0x8006a1e8 +0x8006a21c +0x8006a3fc +0x8006a53c +0x8006a644 +0x8006a880 +0x8006ac14 +0x8006ac8c +0x8006accc +0x8006ad64 +0x8006ae08 +0x8006ae18 +0x8006ae58 +0x8006ae78 +0x8006ae98 +0x8006b060 +0x8006b274 +0x8006b284 +0x8006b298 +0x8006b64c +0x8006b95c +0x8006beac +0x8006bff8 +0x8006c0e4 +0x8006c118 +0x8006c120 +0x8006c38c +0x8006c450 +0x8006c54c +0x8006ca18 +0x8006ca5c +0x8006cb2c +0x8006d010 +0x8006d210 +0x8006d300 +0x8006d840 +0x8006d8ac +0x8006d8fc +0x8006d94c +0x8006d9e8 +0x8006da9c +0x8006e020 +0x8006e0e4 +0x8006e0f4 +0x8006e108 +0x8006e118 +0x8006e148 +0x8006e1cc +0x8006e23c +0x8006e26c +0x8006e29c +0x8006e2cc +0x8006e6a8 +0x8006e6c8 +0x8006e7c0 +0x8006e83c +0x8006ead0 +0x8006ec9c +0x8006ed6c +0x8006ee28 +0x8006f0b8 +0x8006f140 +0x8006f1a8 +0x8006f1f0 +0x8006f294 +0x8006f2d0 +0x8006f330 +0x8006f35c +0x8006f388 +0x8006f3e8 +0x8006f40c +0x8006f488 +0x8006f498 +0x8006f4ec +0x8006f7b4 +0x8006fabc +0x8006fb38 +0x8006fb5c +0x8006fd64 +0x8006fe34 +0x8006feb4 +0x8006ff40 +0x80070438 +0x80070908 +0x80070958 +0x800709d0 +0x800709f4 +0x80070b90 +0x80070ba0 +0x80070d9c +0x80070dfc +0x80070e54 +0x80070efc +0x80070f28 +0x80070f40 +0x800712d4 +0x800713a4 diff --git a/jp/stage_prototype.py b/jp/stage_prototype.py new file mode 100644 index 0000000..ce32446 --- /dev/null +++ b/jp/stage_prototype.py @@ -0,0 +1,37 @@ +"""Add Japan to a NEW verified RC2 package copy. Existing EU files stay intact.""" +import argparse,hashlib,json,shutil,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'eu'),str(ROOT/'psxrecomp/tools')] +from toolchain import executable +import aot_overlay_pipeline as pipeline + +def main(): + p=argparse.ArgumentParser();p.add_argument('--base',type=Path,required=True);p.add_argument('--output',default='dist/TombaRecomp-EU-Japan-Prototype');a=p.parse_args();base=a.base.resolve();out=(ROOT/a.output).resolve() + assert out.is_relative_to(ROOT) and not out.exists() + manifest=json.loads((base/'PACKAGE_AUDIT.json').read_text())['files_sha256'] + actual={f.relative_to(base).as_posix():hashlib.sha256(f.read_bytes()).hexdigest() for f in base.rglob('*') if f.is_file() and f!=base/'PACKAGE_AUDIT.json'} + assert actual==manifest,'Base must be an untouched clean RC2 package' + shutil.copytree(base,out);target=out/'backends/SLPS-01144';target.mkdir() + build=ROOT/'build-jp/player/native';assert 'PSX_DEBUG_TOOLS:BOOL=OFF' in (build/'CMakeCache.txt').read_text() + shutil.copy2(build/'TombaRecompJP.exe',target/'TombaRecompJP.exe') + for folder in ['assets','bios']:shutil.copytree(build/folder,target/folder) + shutil.copytree(base/'overlay_toolchain',target/'overlay_toolchain') + shutil.copytree(ROOT/'jp/seeds',target/'seeds') + config=(ROOT/'jp/game.toml').read_text();config=config.replace('exe = "../disc/jp/SLPS_011.44"','exe = "user-data/SLPS_011.44"').replace('disc = "../disc/jp/disc.cue"\n','').replace('out_dir = "../build-jp/generated"','out_dir = "cache/generated"') + (target/'game.toml').write_text(config) + receipt=json.loads((ROOT/'build-jp/aot/audit.json').read_text()) + pipeline.stage(ROOT/'build-jp/qa/native/cache',target,receipt) + # Strip debug paths only on NEW copied Japan files; never alter EU binaries. + for f in [target/'TombaRecompJP.exe',*target.glob('cache/**/*.dll')]:subprocess.run([executable('llvm-strip.exe'),'--strip-debug',str(f)],check=True) + (target/'AOT_CACHE_AUDIT.json').write_text(json.dumps({k:receipt[k] for k in ['game_id','cache_tag','published_pairs','manifest_rows','all_pairs_valid','all_guards_match_known_input_bytes','full_static_coverage_proven']},indent=2)+'\n') + catalog='\n[[profile]]\nid = "SLPS-01144"\nsize = 289430064\nsha256 = "fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b"\nbackend = "backends/SLPS-01144/TombaRecompJP.exe"\n' + with (out/'regional-launch.toml').open('a') as f:f.write(catalog) + shutil.copy2(ROOT/'jp/TESTANLEITUNG_DE.md',out/'JAPAN_TEST_DE.md') + if (ROOT/'jp/TEST_REPORT.json').is_file():shutil.copy2(ROOT/'jp/TEST_REPORT.json',out/'JAPAN_TEST_REPORT.json') + (out/'Start-TombaRecomp.cmd').write_text('@echo off\r\ncd /d "%~dp0"\r\nstart "" "%~dp0TombiRecompEU.exe"\r\n',encoding='ascii') + for name,digest in manifest.items(): + if name!='regional-launch.toml':assert hashlib.sha256((out/name).read_bytes()).hexdigest()==digest,name + subprocess.run([sys.executable,'-I','-B',str(ROOT/'europe/audit.py'),'--path',str(out),'--kind','prototype','--output',str(out/'PACKAGE_AUDIT.json')],check=True) + print(out,flush=True) +if __name__=='__main__':main() diff --git a/jp/stage_regional.py b/jp/stage_regional.py new file mode 100644 index 0000000..b0e24f4 --- /dev/null +++ b/jp/stage_regional.py @@ -0,0 +1,60 @@ +"""Stage four backends using the existing common launcher, never user installs.""" +import argparse,hashlib,json,shutil,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'eu'),str(ROOT/'psxrecomp/tools')] +from toolchain import executable +import aot_overlay_pipeline as pipeline + +def main(): + p=argparse.ArgumentParser();p.add_argument('--output',default='dist/TombaRecomp-Regional-Japan-RC1');p.add_argument('--enhancements',action='store_true');a=p.parse_args() + out=(ROOT/a.output).resolve();assert out.is_relative_to(ROOT) and not out.exists() + base=ROOT/'dist/TombiRecomp-Europe-Windows-x64-RC2' + manifest=json.loads((base/'PACKAGE_AUDIT.json').read_text())['files_sha256'] + assert {f.relative_to(base).as_posix():hashlib.sha256(f.read_bytes()).hexdigest() for f in base.rglob('*') if f.is_file() and f!=base/'PACKAGE_AUDIT.json'}==manifest + shutil.copytree(base,out) + profiles=[] + for region,filename,build,config,receipt_path,cache,seeds in [ + ('SLPS-01144','TombaRecompJP.exe',ROOT/('build-jp/rc2/player/native' if a.enhancements else 'build-jp/full/player/native'),ROOT/('jp/pc.toml' if a.enhancements else 'jp/game.toml'),ROOT/('build-jp/rc2/aot/audit.json' if a.enhancements else 'build-jp/full/aot-pgxp/audit.json'),ROOT/('build-jp/rc2/qa/native/cache' if a.enhancements else 'build-jp/full/qa/native/cache'),ROOT/'jp/seeds'), + ('SCUS-94236','Tomba__Recompiled.exe',ROOT/'build-regional/us-player/native',ROOT/'packaging/release/game.toml',ROOT/'build-regional/us-audit-full/audit.json',ROOT/'build-regional/us-cache-copy',ROOT/'seeds')]: + assert 'PSX_DEBUG_TOOLS:BOOL=OFF' in (build/'CMakeCache.txt').read_text() + target=out/'backends'/region;target.mkdir();shutil.copy2(build/filename,target/filename) + for folder in ['assets','bios','mods']:shutil.copytree(build/folder,target/folder) + if region=='SLPS-01144': + # A staged build may retain an old experimental manifest after CMake + # regeneration. Never offer the unverified Japanese hybrid path. + candidate=target/'mods/bundled/tomba.enhancement.hybrid-controller' + if candidate.exists():shutil.rmtree(candidate) + for personal in ['state.toml','settings.toml']:assert not (target/'mods'/personal).exists() + shutil.copytree(base/'overlay_toolchain',target/'overlay_toolchain');(target/'seeds').mkdir() + for name in (['boot.txt'] if region=='SLPS-01144' else ['ghidra_funcs.txt','tomba_bios_thunks.txt']): + # Seed comments are developer provenance, sometimes with old local paths. + # Only the actual parser input belongs in the portable runtime package. + lines=(seeds/name).read_text().splitlines() + (target/'seeds'/name).write_text('\n'.join(line for line in lines if not line.lstrip().startswith('#'))+'\n') + text=config.read_text() + if region=='SLPS-01144': + text=text.replace('exe = "../disc/jp/SLPS_011.44"','exe = "user-data/SLPS_011.44"').replace('disc = "../disc/jp/disc.cue"\n','').replace('out_dir = "../build-jp/generated"','out_dir = "cache/generated"').replace('Japan Prototype','Japan').replace('SLPS-01144 Prototype','SLPS-01144') + text=text.replace('out_dir = "../build-jp/rc2/generated/boot"','out_dir = "cache/generated"') + size=289430064;digest='fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b';pairs=24 + else: + text=text.replace('memcard_dir = "saves"','memcard_dir = "saves/SCUS-94236"').replace('disc = "tomba/tomba.cue"\n','') + text+='\n[prepare_disc]\nboot_exe = "SCUS_942.36"\nknown_sizes = [280193760]\nknown_md5 = ["c80636a28de4abcfa34f9e57fb3b043d"]\nknown_sha1 = ["c259ec7ff6ef4163913991f4e4db2eff71702818"]\n' + inventory=json.loads((ROOT/'build-regional/us-audit-full/runtime-input-inventory.json').read_text());digest=inventory['original_disc_sha256'];size=(ROOT/'disc/Tomba! (USA).bin').stat().st_size;pairs=176 + (target/'game.toml').write_text(text) + receipt=json.loads(receipt_path.read_text());assert receipt['published_pairs']==pairs and receipt['all_pairs_valid'] and receipt['all_guards_match_known_input_bytes'] + pipeline.stage(cache,target,receipt) + for f in [target/filename,*target.glob('cache/**/*.dll')]:subprocess.run([executable('llvm-strip.exe'),'--strip-debug',str(f)],check=True) + (target/'AOT_CACHE_AUDIT.json').write_text(json.dumps({k:receipt[k] for k in ['game_id','cache_tag','published_pairs','manifest_rows','all_pairs_valid','all_guards_match_known_input_bytes','full_static_coverage_proven']},indent=2)+'\n') + profiles.append(f'\n[[profile]]\nid = "{region}"\nsize = {size}\nsha256 = "{digest}"\nbackend = "backends/{region}/{filename}"\n') + with (out/'regional-launch.toml').open('a') as f:f.write(''.join(profiles)) + (out/'Start-TombaRecomp.cmd').write_text('@echo off\r\ncd /d "%~dp0"\r\nstart "" "%~dp0TombiRecompEU.exe"\r\n',encoding='ascii') + shutil.copy2(ROOT/'jp/PORT_STATUS_EN.md',out/'JAPAN_HANDOFF_EN.md') + if a.enhancements:shutil.copy2(ROOT/'jp/RC2_MOD_REVIEW_EN.md',out/'RC2_MOD_REVIEW_EN.md') + for name,digest in manifest.items(): + if name!='regional-launch.toml':assert hashlib.sha256((out/name).read_bytes()).hexdigest()==digest,name + shutil.copy2(ROOT/'jp/REGIONAL_README_EN.md',out/'README_EN.md') + shutil.copy2(ROOT/'jp/MANUAL_RELEASE_TESTS_DE.md',out/'JAPAN_TEST_DE.md') + shutil.copy2(ROOT/'jp/FEATURES_EN.md',out/'FEATURES.md') + print(out,flush=True) +if __name__=='__main__':main() diff --git a/jp/test_codec.py b/jp/test_codec.py new file mode 100644 index 0000000..be15378 --- /dev/null +++ b/jp/test_codec.py @@ -0,0 +1,26 @@ +"""Compare the shared GAM decoder with original Japanese MIPS instructions. + +This proves codec equivalence only, not a seamless-loading integration. +""" +import hashlib,json,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'tools'),str(ROOT/'psxrecomp/tools')] +from seamless_asset_probe import DecoderOracle,decode_gam +from aot_overlay_pipeline import Disc + +def main(): + disc=Disc(ROOT/'disc/jp/disc.cue');exe=disc.read('SLPS_011.44') + assert hashlib.sha1(exe).hexdigest()=='07d577698ad31ec1a69b9d7b80a7c3f90b650888' + oracle=DecoderOracle(exe,entry=0x8003C8C4);cases=[] + for name in sorted(disc.files): + if not name.endswith('.GAM'):continue + encoded=disc.read(name) + if encoded[:4]!=b'GAM\0':continue + decoded=decode_gam(encoded);oracle.verify(encoded,decoded) + cases.append(dict(file=name,decoded_sha256=hashlib.sha256(decoded).hexdigest())) + assert len(cases)==638,len(cases) + report=dict(result='passed',original_mips_entry='0x8003C8C4',destination_fill_patterns=2,files=len(cases),seamless_loading_proven=False,cases=cases) + out=ROOT/'build-jp/full/codec-differential.json';out.parent.mkdir(parents=True,exist_ok=True);out.write_text(json.dumps(report,indent=2)+'\n') + print('PASS original Japanese MIPS/shared decoder:',len(cases),'files',flush=True) +if __name__=='__main__':main() diff --git a/jp/test_codec_state.py b/jp/test_codec_state.py new file mode 100644 index 0000000..4168ca9 --- /dev/null +++ b/jp/test_codec_state.py @@ -0,0 +1,24 @@ +"""Check actual common C++ decoder metadata against original Japanese MIPS.""" +import csv,hashlib,json,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'build-jp/full/test-deps'),str(ROOT/'tools'),str(ROOT/'psxrecomp/tools')] +from seamless_asset_probe import DecoderOracle,decode_gam +from aot_overlay_pipeline import Disc + +def main(): + disc=Disc(ROOT/'disc/jp/disc.cue');exe=disc.read('SLPS_011.44');assert hashlib.sha1(exe).hexdigest()=='07d577698ad31ec1a69b9d7b80a7c3f90b650888' + oracle=DecoderOracle(exe,entry=0x8003c8c4);seen={};count=0 + for row in csv.DictReader((ROOT/'build-jp/rc2-analysis/codec-states.csv').open()): + raw=disc.read(row['file']);key=hashlib.sha256(raw).hexdigest() + if key not in seen: + decoded=decode_gam(raw);oracle.verify(raw,decoded) + bit=bytes(oracle.uc.mem_read(0x9dbd0,1))[0];flags=struct.unpack(' +#include +#include +#include +#include "cpu_state.h" +#include "eu_widescreen.h" +static unsigned char image[1024*1024];static size_t size;static int started,calls; +static PSXModFunctionEntryCallback callbacks[3];static uint32_t addresses[3];static unsigned count; +int psx_mod_game_started(void){return started;} +uint32_t psx_mod_read_word(uint32_t pc){size_t a=pc-0x80010000u+2048;assert(a+4<=size);return (uint32_t)image[a]|(uint32_t)image[a+1]<<8|(uint32_t)image[a+2]<<16|(uint32_t)image[a+3]<<24;} +int psx_mod_register_function_entry_plugin(const char*n,uint32_t a,PSXModFunctionEntryCallback c){(void)n;assert(count<3);addresses[count]=a;callbacks[count++]=c;return 1;} +static void callback(CPUState*c,uint32_t a){(void)c;(void)a;calls++;} +int main(int argc,char**argv){ + assert(argc==2);FILE*f=fopen(argv[1],"rb");assert(f);size=fread(image,1,sizeof image,f);fclose(f); + assert(eu_ws_register_entry_plugin("tomba.widescreen",0x800636a4u,callback)); + assert(eu_ws_register_entry_plugin("tomba.widescreen",0x8005e08cu,callback)); + assert(eu_ws_register_entry_plugin("tomba.widescreen",0x8005a184u,callback));assert(count==3); + assert(!eu_ws_refresh());started=1;assert(eu_ws_refresh()); + assert(eu_ws_address(0x8009bcc8u)==0x8009e508u);assert(eu_ws_address(0x800a38b8u)==0x800a6098u); + assert(eu_ws_address(0x8005e08cu)==0x8005af80u);assert(eu_ws_address(0x80023608u)==0x800219f0u); + assert(eu_ws_code_key(0x8004bac4u)==0x8004e4c4u); + for(unsigned i=0;i>26==3;return 0x80000000|((w&0x3ffffff)<<2) + report={'implemented':False,'source':'original Japan disc, original callers, existing psx_tomba symbol aliases', + 'event_predicate':{'eu1':'0x80020a78','jp':hex(jal(0x800576cc)),'original_caller':'0x800576cc','return_continuation':'0x800576d4','different_body':True}, + 'hud_parent':{'eu1':'0x80058680','jp':'0x8004ba68','stack_frame_bytes':0x90, + 'callee_returns':[{'eu1':hex(a),'jp':hex(b),'jp_callee':hex(jal(b-8))} for a,b in [(0x800586dc,0x8004bac4),(0x800586ec,0x8004bad4),(0x80058700,0x8004bae4),(0x80058710,0x8004baf4)]]}, + 'overlay_link':{'jp':'0x800e9af8','original_materialization':['0x8002025c','0x80020260']},'overlays':[]} + assert word(0x8002025c)==0x3c06800f and word(0x80020260)==0x24c69af8 + assert word(0x8004ba68)==0x27bdff70 + for area,sites in [(0,[0x801175cc,0x801176c4]),(1,[0x80116b04,0x80116bec])]: + src=e.read(f'SYSTEM/X0{area:02d}.BIN');dst=d.read(f'SYSTEM/X{area:02d}.BIN');dw=[mask(w) for w in struct.unpack('<%dI'%(len(dst)//4),dst[:len(dst)//4*4])] + for site in sites: + at=site-0x800e8050;pat=[mask(w) for w in struct.unpack('<16I',src[at-32:at+32])] + matches=[i for i in range(len(dw)-16) if dw[i:i+16]==pat] + row={'area':area,'eu1':hex(site),'candidates':[hex(0x800e9af8+(i+8)*4) for i in matches]} + if len(matches)==1: + off=(matches[0]+8)*4;row.update(word=hex(struct.unpack_from('=8)return 0;hooks[hook_count].key=key;hooks[hook_count++].callback=callback;switch(key){'] + for key in (0x800636a4,0x8005e08c,0x8005a184): + impl.append('case '+hex(key)+'u:') + for value in sorted({r[key] for r in rows}):impl.append('(void)psx_mod_register_function_entry_plugin(name,'+hex(value)+'u,entry);') + impl.append('return 1;') + impl+=['default:return 0;}}'];(out/'eu_widescreen.c').write_text('\n'.join(impl)+'\n') + for name,sha in shared.SOURCES.items(): + source=(ROOT/'src/mods'/name).read_text();assert hashlib.sha256(source.encode()).hexdigest()==sha + pool=re.search(r'static const Pool pools\[\] = \{.*?\n\};',source,re.S);descriptor=pool.group(0) if pool else None + if descriptor:source=source.replace(descriptor,'/* POOL_DESCRIPTORS */') + registrations={} + cases={} + def case(match): + key='WS_CASE_'+match.group(1)[2:-1];cases[key]=match.group(1);return 'case '+key+':' + source=re.sub(r'case (0x[0-9a-fA-F]+u):',case,source) + if cases:source=source.replace('switch (ra)', 'switch (eu_ws_code_key(ra))') + def registration(match): + key='WS_ENTRY_'+match.group(2)[2:-1] + registrations[key]=match.group(2) + return 'eu_ws_register_entry_plugin('+match.group(1)+key+match.group(3) + source=re.sub(r'psx_mod_register_function_entry_plugin\((\s*"[^"]+",\s*)(0x[0-9a-fA-F]+u)(,\s*\w+\))',registration,source) + def replace(match): + old=int(match.group(1),16) + if old in rows[0]:return f'eu_ws_address(0x{old:08x}u)' + if old in (0x8000fe00,0x80098000,0x801fffb8,0x801ffffc,0x80200000):return match.group(0) + raise ValueError('Unmapped literal '+match.group(0)) + source=re.sub(r'0x(80[0-9a-fA-F]{6})u\b',replace,source) + for key,value in registrations.items():source=source.replace(key,value) + for key,value in cases.items():source=source.replace(key,value) + source=source.replace('0x0C00B3BEu',overlay_call).replace('0x0C008B91u',script_call).replace('0x22E44u',hex(codes[0x80022e44]&0x1fffffff)+'u') + if descriptor:source=source.replace('/* POOL_DESCRIPTORS */',descriptor).replace('pools[p].base','eu_ws_address(pools[p].base)') + source=source.replace('if (!tomba_widescreen_enabled) return;','if (!tomba_widescreen_enabled || !eu_ws_refresh()) { psx_mod_set_widescreen_view_anchor(0,0); return; }') + source=source.replace('if (!enabled ||','if (!eu_ws_refresh() || !enabled ||') + source=source.replace(' tomba_widescreen_scripts_tick();',' if (!eu_ws_refresh()) { psx_mod_set_widescreen_view_anchor(0,0); return; }\n tomba_widescreen_scripts_tick();') + (out/name).write_text('#include "eu_widescreen.h"\n'+source) + diff --git a/jp/widescreen_profile.py b/jp/widescreen_profile.py new file mode 100644 index 0000000..854605e --- /dev/null +++ b/jp/widescreen_profile.py @@ -0,0 +1,57 @@ +"""Experimental Japan bindings; emit the same shared policy as EU2.""" +import argparse,hashlib,importlib.util,json,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path[:0]=[str(ROOT/'eu'),str(ROOT/'eu2'),str(ROOT/'psxrecomp/tools')] +from aot_overlay_pipeline import Disc,check_disc +spec=importlib.util.spec_from_file_location('eu2_ws_emitter',ROOT/'jp/widescreen_emitter.py') +emitter=importlib.util.module_from_spec(spec);spec.loader.exec_module(emitter) +shared=emitter.shared +def generate(capture=False,output=ROOT/'build-jp/rc2/generated'): + contract=ROOT/'jp/widescreen-contract.json' + disc=Disc(ROOT/'disc/jp/disc.cue') + check_disc(dict(disc_hashes={'sha256':'fb5b8ea9e3bd328e88a6b8b8ecb0b493d0dda5a9b5f732a7a6e4c9b146d9cd2b'}),disc,ROOT/'disc/jp/disc.cue') + raw=disc.read('SLPS_011.44');assert hashlib.sha1(raw).hexdigest()=='07d577698ad31ec1a69b9d7b80a7c3f90b650888' + body=raw[2048:] + def word(pc):return struct.unpack_from('>26==15 and hi>>16&31==lo>>21&31 and lo>>26 in (9,37) + row[old]=((hi&65535)<<16)+(lo&65535)-(65536 if lo&32768 else 0) + row[0x8009bcca]=row[0x8009bcc8]+2 + row.update({old:mapping[pc] for old,pc in shared.TABLES.items()}) + assert row[0x8009bcc8]==0x8009e508 + assert word(row[0x8007f0cc])==row[0x8007eefc] and word(row[0x8007eefc])==0x1f800358 + assert word(code[0x8005e08c])==0x10a00004 + assert word(code[0x8003d8b8]-8)==0x0c00848b and word(code[0x8003e828]-8)==0x0c00848b + assert word(0x800576cc)==0x0c00867c and code[0x80023608]==0x800219f0 + assert word(0x8004ba68)==0x27bdff70 + for site,target in [(0x8004bac4,0x8004bb8c),(0x8004bad4,0x8004bd10),(0x8004bae4,0x8004befc),(0x8004baf4,0x8004c2f0)]: + assert word(site-8)==0x0c000000|((target&0xfffffff)>>2) + assert word(0x8002025c)==0x3c06800f and word(0x80020260)==0x24c69af8 + overlays={0x801168f4:(0,0x8011679c,0x27bdffe8),0x801169ec:(0,0x80116894,0x0c00ac78), + 0x80115e2c:(1,0x80115db0,0x28420b6e),0x80115f14:(1,0x80115e98,0x28620b6d)} + checks={hex(pc):hashlib.sha256(body[pc-0x80010000-32:pc-0x80010000+32]).hexdigest() for pc in sorted(set(mapping.values()))} + for old,(area,pc,expected) in overlays.items(): + image=disc.read(f'SYSTEM/X{area:02d}.BIN');at=pc-0x800e9af8 + assert struct.unpack_from('( std::filesystem::file_time_type::clock::now().time_since_epoch().count()))); @@ -96,6 +96,11 @@ int main(int argc, char **argv) { const TombaAsset *assets = tomba_seamless_prepare(&count); require(assets && count == std::size(catalog) && reads == std::size(catalog), "cold native preparation"); unsigned decoded = 0, gam_index = ~0u; + std::ofstream codec_states; + if (argc == 4) { + codec_states.open(argv[3]); require(bool(codec_states), "codec state output"); + codec_states << "file,declared,state,decoded_len\n"; + } for (unsigned i = 0; i < count; i++) { const auto &a = assets[i]; const auto &c = catalog[i]; @@ -107,11 +112,13 @@ int main(int argc, char **argv) { hash(a.decoded, a.decoded_len) == c.decoded_sha256 && a.declared && a.declared <= a.decoded_len, "decoded GAM matches the original decoder"); ++decoded; + if (codec_states.is_open()) codec_states << c.path << ',' << a.declared << ',' << a.codec_state << ',' << a.decoded_len << '\n'; if (gam_index == ~0u) gam_index = i; } else { require(!a.decoded && !a.decoded_len, "raw asset has no decoded output"); } } + if (codec_states.is_open()) codec_states.close(); reads = 0; available = false; assets = tomba_seamless_prepare(&count); require(assets && reads == 0, "warm launch uses the verified pack");