Ore! Tomba (Japan) port

.gitignore

@@ -66,3 +66,4 @@ tools/_toolchain_cache/
 /dist/
 /owner-playtest/
 /launch-temp/
+jp/saves/

eu/seamless_catalog.py

@@ -20,25 +20,27 @@ def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--oracle', action='store_true',
                         help='Compare unique streams with the original EU MIPS decoder, two fills')
-    parser.add_argument('--variant',choices=['eu1','eu2'],default='eu1')
+    parser.add_argument('--variant',choices=['eu1','eu2','jp'],default='eu1')
+    parser.add_argument('--output',type=Path,help='Private generated catalog directory')
     parser.add_argument('--oracle-limit',type=int,help='Explicit bounded original-MIPS sample; never claims full corpus')
     args = parser.parse_args()
     started = time.monotonic()
-    cue = ROOT/('disc/eu2/disc.cue' if args.variant=='eu2' else 'disc/Tombi! (Europe) (En,Fr,De,It,Nl) (Rev 1).cue')
+    cue = ROOT/('disc/jp/disc.cue' if args.variant=='jp' else 'disc/eu2/disc.cue' if args.variant=='eu2' else 'disc/Tombi! (Europe) (En,Fr,De,It,Nl) (Rev 1).cue')
     disc = Disc(cue)
     eu2_profile = None
     if args.variant=='eu2':
         sys.path.insert(0,str(ROOT/'eu2'))
         from revisions import PROFILES, active, source
         eu2_profile=PROFILES[active()]
-    check_disc(dict(disc_hashes={'sha1': eu2_profile['sha1'] if eu2_profile else
+    check_disc(dict(disc_hashes={'sha1': 'c86baf702303e7e0963b99aff307b1c40e40478a' if args.variant=='jp' else eu2_profile['sha1'] if eu2_profile else
                                '2505d958b4439d04844cadd3f968fc7432a48e00'}), disc, cue)
     oracle = None
     if args.oracle:
-        sys.path.insert(0, str(ROOT/('build-eu2-seamless/python-deps' if args.variant=='eu2' else 'build-eu-seamless/python-deps')))
-        oracle = DecoderOracle(disc.read('EXE/MAIN0.EXE' if args.variant=='eu2' else 'EXE/MAIN2.EXE'),
-                               entry=int(json.loads(source('seamless-contract.json').read_text())['languages'][0]['code']['0x8003bafc']['address'],16) if eu2_profile else 0x8003BAFC)
-    out = ROOT/('build-eu2-seamless/generated' if args.variant=='eu2' else 'build-eu-seamless/generated')
+        sys.path.insert(0, str(ROOT/('build-jp/full/test-deps' if args.variant=='jp' else 'build-eu2-seamless/python-deps' if args.variant=='eu2' else 'build-eu-seamless/python-deps')))
+        oracle = DecoderOracle(disc.read('SLPS_011.44' if args.variant=='jp' else 'EXE/MAIN0.EXE' if args.variant=='eu2' else 'EXE/MAIN2.EXE'),
+                               entry=0x8003C8C4 if args.variant=='jp' else int(json.loads(source('seamless-contract.json').read_text())['languages'][0]['code']['0x8003bafc']['address'],16) if eu2_profile else 0x8003BAFC)
+    out = (args.output or ROOT/('build-jp/final/generated' if args.variant=='jp' else 'build-eu2-seamless/generated' if args.variant=='eu2' else 'build-eu-seamless/generated')).resolve()
+    assert out.is_relative_to(ROOT)
     out.mkdir(parents=True, exist_ok=True)
     rows, streams, decoded = [], [], {}
     source_bytes = decoded_bytes = gam_count = oracle_count = 0

eu2/widescreen_analysis.py

@@ -0,0 +1,80 @@
+"""Read-only discovery of regional equivalents of the official EU1 hooks.
+
+Candidate discovery is not a runtime authorization. Full raw windows, changed
+instructions and ambiguity are recorded for review before any hook is enabled.
+"""
+import argparse, difflib, hashlib, importlib.util, json, struct
+from pathlib import Path
+
+BASE=0x80010000
+def words(data): return struct.unpack('<%dI'%(len(data)//4),data[:len(data)//4*4])
+def mask(w):
+    op=w>>26
+    if op in (2,3): return w&0xfc000000
+    if op in (1,4,5,6,7,8,9,10,11,12,13,14,15,32,33,34,35,36,37,38,40,41,42,43,46,50,58): return w&0xffff0000
+    return w
+
+def main():
+    p=argparse.ArgumentParser();p.add_argument('--reference-root',type=Path,required=True);a=p.parse_args()
+    root=a.reference_root.resolve()
+    spec=importlib.util.spec_from_file_location('official_ws',root/'eu/widescreen_profile.py')
+    m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
+    reference=(root/'disc/EXE/MAIN0.EXE').read_bytes()[2048:];rw=words(reference)
+    sites=sorted(set(m.CODE.values())|set(m.FIELDS.values())|set(m.TABLES.values())|{0x8004f624,0x8004f628})
+    targets={'eu2-en':root/'disc/eu2/EXE/MAIN0.EXE','eu2-es':root/'disc/eu2/EXE/MAIN5.EXE',
+      'eu2-pt':root/'disc/eu2/EXE/MAIN6.EXE','eu2-sv':root/'disc/eu2/EXE/MAIN7.EXE',
+      'eu2-da':root/'disc/eu2/EXE/MAIN8.EXE','jp':root/'disc/jp/SLPS_011.44'}
+    report={}
+    for name,path in targets.items():
+        raw=path.read_bytes();body=raw[2048:];tw=words(body);normalized=[mask(w) for w in tw]
+        index={}
+        for i in range(len(tw)-8):index.setdefault(tuple(normalized[i:i+8]),[]).append(i)
+        rows=[]
+        for pc in sites:
+            ri=(pc-BASE)//4;found=[];used=0
+            for radius in (16,8,4,2):
+                lo=ri-radius;hi=ri+radius;key=tuple(mask(w) for w in rw[lo:lo+8])
+                candidates=[i+radius for i in index.get(key,[]) if i+2*radius<=len(tw) and normalized[i:i+2*radius]==[mask(w) for w in rw[lo:hi]]]
+                if len(candidates)==1:found=candidates;used=radius;break
+            row={'eu1':hex(pc),'candidates':[hex(BASE+i*4) for i in found],'window_bytes':used*8}
+            if found:
+                ti=found[0];b=body[(ti-used)*4:(ti+used)*4]
+                row.update(address=hex(BASE+ti*4),sha256=hashlib.sha256(b).hexdigest(),
+                  changes=[{'relative':(i-used)*4,'eu1':hex(x),'target':hex(y)} for i,(x,y) in enumerate(zip(rw[ri-used:ri+used],tw[ti-used:ti+used])) if x!=y])
+            rows.append(row)
+        if name.startswith('eu2-'):
+            # Resolve repetitive call sites by their enclosing independently
+            # matched boundaries, never by a whole-region address delta.
+            for row in rows:
+                if 'address' in row:continue
+                pc=int(row['eu1'],16)
+                left=max((r for r in rows if 'address' in r and int(r['eu1'],16)<pc),key=lambda r:int(r['eu1'],16))
+                right=min((r for r in rows if 'address' in r and int(r['eu1'],16)>pc),key=lambda r:int(r['eu1'],16))
+                lo=int(left['eu1'],16);hi=int(right['eu1'],16)
+                delta=int(left['address'],16)-lo
+                if int(right['address'],16)-hi!=delta:continue
+                if [mask(w) for w in rw[(lo-BASE)//4:(hi-BASE)//4]]!=normalized[(lo+delta-BASE)//4:(hi+delta-BASE)//4]:continue
+                target=pc+delta;at=target-BASE
+                row.update(address=hex(target),sha256=hashlib.sha256(body[at-32:at+32]).hexdigest(),window_bytes=64,
+                    anchored_interval=[hex(lo),hex(hi)],changes=[{'relative':d,'eu1':hex(struct.unpack_from('<I',reference,pc-BASE+d)[0]),'target':hex(struct.unpack_from('<I',body,at+d)[0])} for d in range(-32,32,4) if reference[pc-BASE+d:pc-BASE+d+4]!=body[at+d:at+d+4]])
+        if name=='jp':
+            for row in rows:
+                if 'address' in row:continue
+                pc=int(row['eu1'],16)
+                left=max((r for r in rows if 'address' in r and int(r['eu1'],16)<pc),key=lambda r:int(r['eu1'],16))
+                right=min((r for r in rows if 'address' in r and int(r['eu1'],16)>pc),key=lambda r:int(r['eu1'],16))
+                lo=int(left['eu1'],16);hi=int(right['eu1'],16);tl=int(left['address'],16);th=int(right['address'],16)
+                if not 0<hi-lo<0x4000 or not 0<th-tl<0x4000:continue
+                seq=difflib.SequenceMatcher(None,[mask(w) for w in rw[(lo-BASE)//4:(hi-BASE)//4]],normalized[(tl-BASE)//4:(th-BASE)//4],autojunk=False)
+                index=(pc-lo)//4
+                for block in seq.get_matching_blocks():
+                    if block.a<=index<block.a+block.size:
+                        row['aligned_candidate']=hex(tl+(block.b+index-block.a)*4)
+                        row['aligned_block_words']=block.size
+                        row['alignment_boundaries']=[hex(lo),hex(hi),hex(tl),hex(th)]
+                        break
+        report[name]={'exe_sha256':hashlib.sha256(raw).hexdigest(),'rows':rows}
+        print(name,'unique',sum('address' in r for r in rows),'/',len(rows),'missing',[r['eu1'] for r in rows if 'address' not in r],flush=True)
+    out=Path(__file__).resolve().parents[1]/'build-widescreen';out.mkdir(exist_ok=True)
+    (out/'discovery.json').write_text(json.dumps(report,indent=2)+'\n')
+if __name__=='__main__':main()

jp/CMakeLists.txt

@@ -0,0 +1,60 @@
+cmake_minimum_required(VERSION 3.20)
+project(OreTombaJapan LANGUAGES C CXX)
+get_filename_component(TOMBA_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE)
+set(PSXRECOMP_ROOT "${TOMBA_ROOT}/psxrecomp" CACHE PATH "Shared framework")
+set(RECOMP_UI_ROOT "${TOMBA_ROOT}/recomp-ui" CACHE PATH "Shared launcher")
+set(PSX_SETUP_WIZARD OFF CACHE BOOL "No developer wizard" FORCE)
+set(PSXRECOMP_REQUIRE_GAME_C ON CACHE BOOL "Verified native boot required" FORCE)
+include("${PSXRECOMP_ROOT}/runtime/runtime.cmake")
+set(PSX_BUILTIN_MOD_ALLOWLIST psx.enhancement.pgxp psx.presentation.bezel psx.enhancement.hd-textures)
+# The enhanced build carries every Japanese feature: Seamless Loading and FMV
+# Skip (checked loader/movie bindings) and Widescreen (checked shared-renderer
+# adapter, jp/widescreen_profile.py).
+option(TOMBA_JP_ENHANCEMENTS "Seamless Loading, FMV Skip and Widescreen" OFF)
+set(TOMBA_JP_GENERATED_DIR "${TOMBA_ROOT}/build-jp/generated" CACHE PATH "Japanese boot code directory")
+set(jp_mods NONE)
+if(TOMBA_JP_ENHANCEMENTS)
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    get_filename_component(jp_generated "${TOMBA_JP_GENERATED_DIR}" DIRECTORY)
+    execute_process(COMMAND "${Python3_EXECUTABLE}" -I -B "${CMAKE_CURRENT_SOURCE_DIR}/seamless_profile.py" --output "${jp_generated}" COMMAND_ERROR_IS_FATAL ANY)
+    execute_process(COMMAND "${Python3_EXECUTABLE}" -I -B "${CMAKE_CURRENT_SOURCE_DIR}/fmv_profile.py" --output "${jp_generated}" COMMAND_ERROR_IS_FATAL ANY)
+    execute_process(COMMAND "${Python3_EXECUTABLE}" -I -B "${TOMBA_ROOT}/eu/seamless_catalog.py" --variant jp --output "${jp_generated}" COMMAND_ERROR_IS_FATAL ANY)
+    execute_process(COMMAND "${Python3_EXECUTABLE}" -I -B "${CMAKE_CURRENT_SOURCE_DIR}/widescreen_profile.py" --output "${jp_generated}" COMMAND_ERROR_IS_FATAL ANY)
+    set(jp_mods "${CMAKE_CURRENT_SOURCE_DIR}/mods")
+endif()
+psxrecomp_add_game_runtime(tomba-jp
+    PGXP
+    WINDOW_TITLE "Ore! Tomba"
+    DEFAULT_GAME_CONFIG_PATH "game.toml"
+    VERSION_FILE "${TOMBA_ROOT}/VERSION"
+    MAX_PLAYERS 1
+    LAUNCHER_BOXART "${CMAKE_CURRENT_SOURCE_DIR}/launcher_assets/img/boxart.tga"
+    GEN_MARKER "${TOMBA_JP_GENERATED_DIR}/SLPS_011.44_dispatch.c"
+    GEN_FULL_GLOB "${TOMBA_JP_GENERATED_DIR}/SLPS_011.44_full_*.c"
+    PRELOADED_MODS_DIR "${jp_mods}"
+    EXCLUDE_BUILTIN_MODS psx.enhancement.8mb-ram psx.enhancement.cd-speed
+        psx.enhancement.fast-loading)
+set_target_properties(tomba-jp PROPERTIES OUTPUT_NAME "TombaRecompJP")
+if(TOMBA_JP_ENHANCEMENTS)
+    set(jp_ws "${jp_generated}/adapter")
+    target_sources(tomba-jp PRIVATE "${jp_ws}/eu_widescreen.c"
+        "${jp_ws}/tomba_widescreen_plugin.c" "${jp_ws}/tomba_widescreen_residency.c"
+        "${jp_ws}/tomba_widescreen_scripts.c" "${TOMBA_ROOT}/src/mods/tomba_dispatch.c")
+    target_include_directories(tomba-jp PRIVATE "${jp_ws}" "${TOMBA_ROOT}/src/mods")
+    target_sources(tomba-jp PRIVATE "${jp_generated}/jp_loader.c" "${TOMBA_ROOT}/src/mods/tomba_seamless_prepare.cpp" "${TOMBA_ROOT}/src/mods/tomba_skip_fmv_plugin.c")
+    target_include_directories(tomba-jp PRIVATE "${jp_generated}" "${TOMBA_ROOT}/eu/seamless" "${TOMBA_ROOT}/src/mods")
+    set_source_files_properties("${TOMBA_ROOT}/src/mods/tomba_seamless_prepare.cpp" PROPERTIES COMPILE_DEFINITIONS "TOMBA_SEAMLESS_CATALOG_HEADER=\"${jp_generated}/eu_catalog.inc\";TOMBA_SEAMLESS_PACK_FORMAT=\"slps01144-gam-v1\"")
+    if(PSX_DEBUG_TOOLS)
+        add_executable(tomba-jp-loader-tests "${jp_generated}/jp_loader_tests.c")
+        target_include_directories(tomba-jp-loader-tests PRIVATE "${jp_generated}" "${TOMBA_ROOT}/eu/seamless" "${TOMBA_ROOT}/src/mods" "${PSXRECOMP_ROOT}/runtime/include")
+        target_compile_options(tomba-jp-loader-tests PRIVATE -O2 -UNDEBUG -ffunction-sections -fdata-sections)
+        target_link_options(tomba-jp-loader-tests PRIVATE -Wl,--gc-sections)
+        add_executable(tomba-jp-prepare-tests "${TOMBA_ROOT}/tests/test_seamless_prepare.cpp" "${TOMBA_ROOT}/src/mods/tomba_seamless_prepare.cpp" "${PSXRECOMP_ROOT}/runtime/src/mod_resident.cpp" "${PSXRECOMP_ROOT}/runtime/src/psx_sha256.c")
+        target_include_directories(tomba-jp-prepare-tests PRIVATE "${TOMBA_ROOT}/src/mods" "${PSXRECOMP_ROOT}/runtime/include")
+        target_compile_features(tomba-jp-prepare-tests PRIVATE cxx_std_17)
+        target_compile_definitions(tomba-jp-prepare-tests PRIVATE TOMBA_SEAMLESS_CATALOG_HEADER="${jp_generated}/eu_catalog.inc")
+    endif()
+endif()
+# The shared launcher already embeds the UTF-8 application manifest
+# (recomp-ui windows/recomp_utf8.manifest); a second manifest resource would
+# collide with it at link time.