From e292aed3199d0f990f26844b9611c28f0c428dde Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 17:37:25 +0000 Subject: [PATCH] =?UTF-8?q?Foundation:=20=E6=96=87=E5=AD=97=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E5=A4=89=E6=8F=9B=20libcode=20=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=20(#160)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UTF-32BE を基準に、UTF-8（BOM あり/なし）、UTF-16LE/BE、UTF-32LE との cpt_code__to_utf32be / cpt_code_utf32be_to_ を /lib/code.1.wal として提供する。 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01FFAstnZtE3BzBcmKEHnjaW --- foundation/README.md | 9 + foundation/boot/build.py | 9 +- foundation/include/foundation/code.h | 64 ++++++ foundation/lib/code/build.py | 35 +++ foundation/lib/code/code.c | 272 ++++++++++++++++++++++++ foundation/lib/code/code.exports | 13 ++ foundation/tests/code/app.c | 72 +++++++ foundation/tests/code/kernel.c | 20 ++ foundation/tests/code/test_code.py | 269 +++++++++++++++++++++++ foundation/tests/disk/test_disk_boot.py | 2 +- memory/2026-10-24-issue-160-libcode.md | 7 + 11 files changed, 770 insertions(+), 2 deletions(-) create mode 100644 foundation/include/foundation/code.h create mode 100644 foundation/lib/code/build.py create mode 100644 foundation/lib/code/code.c create mode 100644 foundation/lib/code/code.exports create mode 100644 foundation/tests/code/app.c create mode 100644 foundation/tests/code/kernel.c create mode 100644 foundation/tests/code/test_code.py create mode 100644 memory/2026-10-24-issue-160-libcode.md diff --git a/foundation/README.md b/foundation/README.md index 5c058c9..8623f04 100644 --- a/foundation/README.md +++ b/foundation/README.md @@ -269,6 +269,15 @@ src32_testbench /absolute/path/target/foundation-wax/test.wax 42 --cpu pipeline `librle.wal`（`lib/rle`、image ID `rle`、ABI 1.0）は PackBits 形式の RLE で、`/lib/rle.1.wal` として既定のディスクに入ります。heap・libfnd・作業領域を使わず、4 byte 単位でラン検出・コピー・埋め込みを行います。テストは `python3 -m unittest foundation.tests.rle.test_rle`（Clang と `src32_testbench` が必要。後者がなければ Foundation 上のテストは skip）です。 +### 文字コード変換 libcode + +`libcode.wal`（`lib/code`、image ID `code`、ABI 1.0、`/lib/code.1.wal`）は UTF-8（BOM あり／なし）、UTF-16LE／BE、UTF-32LE／BE を変換します。UTF-32BE を基準の形式とし、`cpt_code__to_utf32be` と `cpt_code_utf32be_to_`（`` は `utf8`、`utf8bom`、`utf16le`、`utf16be`、`utf32le`）の対で変換します。他の符号化どうしは UTF-32BE を経由します。引数・戻り値・`*dst_size` は conde と同じ形（`flags` を追加）で、規約は [`include/foundation/code.h`](include/foundation/code.h) にあります。 + +- 不正な列（過長形式、サロゲート、U+10FFFF 超、対にならないサロゲート）は `CONDE_DATA`、末尾で切れた列は `CONDE_INPUT_EOF`。`CODE_REPLACE` を付けると不正な部分（maximal subpart）ごとに U+FFFD を書いて続けます。 +- BOM: `utf8bom` への変換は BOM を必ず付け、からの変換は先頭の BOM を読み飛ばします。他の符号化には `CODE_ADD_BOM`／`CODE_STRIP_BOM`。BOM から符号化を調べるのは `cpt_code_detect`（`FF FE 00 00` は UTF-32LE）。 +- 出力の見積もりは `cpt_code_bound`。heap・libfnd・作業領域は使いません。 +- テストは `python3 -m unittest foundation.tests.code.test_code`（ホストは Clang、Foundation 上は `src32_testbench` も必要）です。 + `tests/wal/` は、`tools/llvm-src32` の `--wal`、`--wax3`、`--import-wal`（[ツールチェーンの README](../tools/llvm-src32/README.md#wax-v3--wal)）で C から作った画像を、カーネルで実行します。`build.py` が `test.base`（WAL）、それを import する `test.math`（WAL、1.2）、両方を import する `test.app`（WAX v3、スロット5と6）を作り、`images.h` に書き出します。`app.c` は、WAL どうしの呼び出し、WAL 内部の関数ポインタ、WAL の静的データがプロセスごとに別であること、オブジェクトの import、文字列定数のアドレス、64-bit の引数と戻り値を確認し、すべて通れば 42 で終了します（起動コードが `proc_exit` を呼ぶ）。 ```sh diff --git a/foundation/boot/build.py b/foundation/boot/build.py index fdac088..156da29 100644 --- a/foundation/boot/build.py +++ b/foundation/boot/build.py @@ -26,7 +26,7 @@ CPUID reports V (doc/foundation_vm_plan.md). Outputs: - root/ /bin/*.wax (the shell and the commands), /lib/fnd.1.wal, /lib/lzma.1.wal, /lib/rle.1.wal and /lib/sgc.1.wal + root/ /bin/*.wax (the shell and the commands), /lib/fnd.1.wal, /lib/lzma.1.wal, /lib/rle.1.wal, /lib/code.1.wal and /lib/sgc.1.wal foundation.wax kernel + module-only FNDBOOT as a block at 0x200000; the host loads it directly (§11 without stage1) and fs mounts IDC slot 0 as "/" kernel.bin kernel raw image for 0x00010000, entered by stage1 @@ -99,6 +99,7 @@ # Commands that import /lib/sgc.1.wal (libsgc) as well as fnd. SGC_COMMANDS = {"chip8"} RLE_DIR = FOUNDATION / "lib/rle" +CODE_DIR = FOUNDATION / "lib/code" FS = [FOUNDATION / "fs" / name for name in ("main.c", "fs2.c", "vfs.c", "server.c", "dsfs.c")] # KAN-38 temporary boot mitigation. Set to 1 to restore the full §10 scan. FS2_FULL_MOUNT_CHECK = 0 @@ -203,6 +204,11 @@ def main(argv=None): rle_build = importlib.util.module_from_spec(rle_spec) rle_spec.loader.exec_module(rle_build) rle_job = pool.submit(rle_build.build, out / "rle", args.clang, args.optimization) + # /lib/code.1.wal: character code conversion (lib/code, WAL "code" 1.0), also self-contained. + code_spec = importlib.util.spec_from_file_location("code_build", CODE_DIR / "build.py") + code_build = importlib.util.module_from_spec(code_spec) + code_spec.loader.exec_module(code_build) + code_job = pool.submit(code_build.build, out / "code", args.clang, args.optimization) kernel_args = ["--origin", hex(KERNEL_LOAD), "--stack-top", hex(KERNEL_STACK_TOP)] if args.stop_ticks is not None: @@ -252,6 +258,7 @@ def main(argv=None): shutil.copyfile(lzma_job.result(), root / "lib/lzma.1.wal") shutil.copyfile(sgc_job.result(), root / "lib/sgc.1.wal") shutil.copyfile(rle_job.result(), root / "lib/rle.1.wal") + shutil.copyfile(code_job.result(), root / "lib/code.1.wal") for job in program_jobs: job.result() modules = [dict(spec, image=spec["image"].result().read_bytes()) for spec in specs] diff --git a/foundation/include/foundation/code.h b/foundation/include/foundation/code.h new file mode 100644 index 0000000..84331c4 --- /dev/null +++ b/foundation/include/foundation/code.h @@ -0,0 +1,64 @@ +#ifndef FOUNDATION_CODE_H +#define FOUNDATION_CODE_H +#include + +/* 文字コード変換。UTF-32BE（コードポイントを 4 byte big endian で並べたもの）を基準の形式とし、 + * 他の符号化との間を cpt_code__to_utf32be と cpt_code_utf32be_to_ で変換します。 + * は utf8、utf8bom、utf16le、utf16be、utf32le。たとえば UTF-16LE から UTF-8 へは + * utf16le_to_utf32be の出力を utf32be_to_utf8 に渡します（UTF-32BE の内容を直接扱う場合は変換が要りません）。 + * 戻り値は CONDE_*（0 が成功）、*dst_size は容量を入力して実長を返し、失敗時は 0 にします（conde と同じ）。 + * 作業領域は使いません。dst と src は重ねないでください。 + * + * int32_t cpt_code__to_utf32be(uint8_t *dst, uint32_t *dst_size, + * const uint8_t *src, uint32_t src_size, uint32_t flags); + * int32_t cpt_code_utf32be_to_(同じ引数); + * + * utf8bom は UTF-8 の BOM 付き。変換先では先頭に EF BB BF を必ず書き、変換元では先頭にあれば読み飛ばす。 + * 出力に BOM を付けたい他の符号化には CODE_ADD_BOM を使う。 + * + * エラー: + * CONDE_DATA 不正な列。UTF-8 の過長形式・サロゲート・U+10FFFF 超、UTF-16 の対にならないサロゲート、 + * UTF-32 のサロゲート・U+10FFFF 超 + * CONDE_INPUT_EOF 末尾で符号化単位が途中で切れている（UTF-8 の列の途中、UTF-16 の上位サロゲートだけ、 + * 端数 byte など） + * CONDE_OUTPUT_EOF 出力が *dst_size に収まらない + * CONDE_PARAMETER flags が不正、ポインタが NULL（src_size == 0 なら src は NULL 可） + * CODE_REPLACE では、不正な部分（Unicode 3.9 の maximal subpart）ごとに U+FFFD を 1 つ書いて続ける。 + * 先頭の U+FEFF は、CODE_STRIP_BOM などで落とさない限り文字として写す（UTF-16LE の FF FE は ZWNBSP になる）。 + * CODE_ADD_BOM と入力側の U+FEFF が重なると BOM が 2 つになる。BOM を整えるなら CODE_STRIP_BOM と組み合わせる。 */ +#define CODE_REPLACE 1u /* 不正な入力を U+FFFD に置き換える（なければ CONDE_DATA／CONDE_INPUT_EOF） */ +#define CODE_STRIP_BOM 2u /* 入力の先頭の U+FEFF を捨てる（utf8bom の変換元は指定なしでも読み飛ばす） */ +#define CODE_ADD_BOM 4u /* 出力の先頭に BOM を付ける（utf8bom の変換先は指定なしでも付く） */ + +#define CPT_CODE_DECLARE(enc) \ + int32_t cpt_code_##enc##_to_utf32be(uint8_t *dst, uint32_t *dst_size, const uint8_t *src, \ + uint32_t src_size, uint32_t flags); \ + int32_t cpt_code_utf32be_to_##enc(uint8_t *dst, uint32_t *dst_size, const uint8_t *src, \ + uint32_t src_size, uint32_t flags); +CPT_CODE_DECLARE(utf8) +CPT_CODE_DECLARE(utf8bom) +CPT_CODE_DECLARE(utf16le) +CPT_CODE_DECLARE(utf16be) +CPT_CODE_DECLARE(utf32le) +#undef CPT_CODE_DECLARE + +/* 符号化の番号（cpt_code_bound と cpt_code_detect が使う）。 */ +enum { + CODE_UTF8 = 0, + CODE_UTF8_BOM = 1, + CODE_UTF16LE = 2, + CODE_UTF16BE = 3, + CODE_UTF32LE = 4, + CODE_UTF32BE = 5 +}; + +/* from から to へ変換した出力が（flags によらず）必ず収まる byte 数。BOM 分を含む。片方が UTF-32BE の + * 変換用だが、どの組み合わせでも使える。表せない場合や符号化が不正な場合は 0xFFFFFFFF。 */ +uint32_t cpt_code_bound(uint32_t src_size, uint32_t from, uint32_t to); + +/* 先頭の BOM から符号化を判別する。BOM がなければ CODE_UTF8 を返す。bom_size（NULL 可）には BOM の長さ + * （なければ 0）を返す。判別できるのは CODE_UTF8_BOM、CODE_UTF16LE／BE、CODE_UTF32LE／BE。 + * FF FE 00 00 は UTF-16LE の BOM と U+0000 でも解釈できるが UTF-32LE とする。src が NULL（src_size > 0）なら負の値。 + * 判別した符号化の関数に src + *bom_size を渡す（utf8bom は BOM ごと渡してよい）。 */ +int32_t cpt_code_detect(const uint8_t *src, uint32_t src_size, uint32_t *bom_size); +#endif diff --git a/foundation/lib/code/build.py b/foundation/lib/code/build.py new file mode 100644 index 0000000..ccbec20 --- /dev/null +++ b/foundation/lib/code/build.py @@ -0,0 +1,35 @@ +"""Build the character code conversion library as a Foundation WAL.""" +import argparse +from pathlib import Path +import subprocess +import sys + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] + + +def build(out, clang=None, optimization="-O2"): + out = Path(out).resolve() + out.mkdir(parents=True, exist_ok=True) + command = [sys.executable, str(ROOT / "tools/llvm-src32/driver.py"), + str(HERE / "code.c"), + "-I", str(ROOT / "foundation/include"), optimization, + "--wal", "--image-id", "code", "--abi-version", "1.0", + "--exports", str(HERE / "code.exports"), "-o", str(out / "libcode.wal")] + if clang: + command += ["--clang", clang] + subprocess.run(command, check=True) + return out / "libcode.wal" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out", type=Path) + parser.add_argument("--clang") + parser.add_argument("--optimization", choices=("-O0", "-O1", "-O2"), default="-O2") + args = parser.parse_args() + print(build(args.out, args.clang, args.optimization)) + + +if __name__ == "__main__": + main() diff --git a/foundation/lib/code/code.c b/foundation/lib/code/code.c new file mode 100644 index 0000000..50cd9b9 --- /dev/null +++ b/foundation/lib/code/code.c @@ -0,0 +1,272 @@ +/* 文字コード変換（include/foundation/code.h）。コードポイントを 1 つずつ読んで書く。 + * heap・libfnd・作業領域は使わず、読み書きは byte 単位なので整列を問わない。 */ +#include + +#define REPLACEMENT 0xFFFDu +#define BOM 0xFEFFu +#define KNOWN_FLAGS (CODE_REPLACE | CODE_STRIP_BOM | CODE_ADD_BOM) + +/* 次のコードポイントを読む。戻り値は CONDE_OK、CONDE_DATA（不正。*i は maximal subpart だけ進む）、 + * CONDE_INPUT_EOF（末尾で途中切れ。*i は n まで進む）。 */ +static int32_t next_utf8(const uint8_t *s, uint32_t n, uint32_t *i, uint32_t *cp) { + uint32_t p = *i, b = s[p++], lo = 0x80, hi = 0xBF, need; + if (b < 0x80) { + *i = p; + *cp = b; + return CONDE_OK; + } + if (b >= 0xC2 && b <= 0xDF) { + need = 1; + *cp = b & 0x1F; + } else if (b >= 0xE0 && b <= 0xEF) { + need = 2; + *cp = b & 0x0F; + if (b == 0xE0) + lo = 0xA0; + else if (b == 0xED) + hi = 0x9F; + } else if (b >= 0xF0 && b <= 0xF4) { + need = 3; + *cp = b & 0x07; + if (b == 0xF0) + lo = 0x90; + else if (b == 0xF4) + hi = 0x8F; + } else { + *i = p; + return CONDE_DATA; + } + for (; need; --need, lo = 0x80, hi = 0xBF) { + if (p >= n) { + *i = p; + return CONDE_INPUT_EOF; + } + b = s[p]; + if (b < lo || b > hi) { + *i = p; + return CONDE_DATA; + } + *cp = *cp << 6 | (b & 0x3F); + ++p; + } + *i = p; + return CONDE_OK; +} + +static uint32_t unit16(const uint8_t *s, int big) { + return big ? (uint32_t)s[0] << 8 | s[1] : (uint32_t)s[1] << 8 | s[0]; +} + +static int32_t next_utf16(const uint8_t *s, uint32_t n, uint32_t *i, uint32_t *cp, int big) { + uint32_t p = *i, u, v; + if (n - p < 2) { + *i = n; + return CONDE_INPUT_EOF; + } + u = unit16(s + p, big); + p += 2; + if (u < 0xD800 || u > 0xDFFF) { + *i = p; + *cp = u; + return CONDE_OK; + } + if (u >= 0xDC00) { /* 下位サロゲートだけ */ + *i = p; + return CONDE_DATA; + } + if (n - p < 2) { /* 上位サロゲートの後に続きがない（1 byte の端数を含む） */ + *i = n; + return CONDE_INPUT_EOF; + } + v = unit16(s + p, big); + if (v < 0xDC00 || v > 0xDFFF) { + *i = p; + return CONDE_DATA; + } + *i = p + 2; + *cp = 0x10000u + ((u - 0xD800) << 10) + (v - 0xDC00); + return CONDE_OK; +} + +static int32_t next_utf32(const uint8_t *s, uint32_t n, uint32_t *i, uint32_t *cp, int big) { + uint32_t p = *i, v; + if (n - p < 4) { + *i = n; + return CONDE_INPUT_EOF; + } + v = big ? (uint32_t)s[p] << 24 | (uint32_t)s[p + 1] << 16 | (uint32_t)s[p + 2] << 8 | s[p + 3] + : (uint32_t)s[p + 3] << 24 | (uint32_t)s[p + 2] << 16 | (uint32_t)s[p + 1] << 8 | s[p]; + *i = p + 4; + if (v > 0x10FFFF || (v >= 0xD800 && v <= 0xDFFF)) + return CONDE_DATA; + *cp = v; + return CONDE_OK; +} + +static int32_t next(uint32_t enc, const uint8_t *s, uint32_t n, uint32_t *i, uint32_t *cp) { + switch (enc) { + case CODE_UTF16LE: return next_utf16(s, n, i, cp, 0); + case CODE_UTF16BE: return next_utf16(s, n, i, cp, 1); + case CODE_UTF32LE: return next_utf32(s, n, i, cp, 0); + case CODE_UTF32BE: return next_utf32(s, n, i, cp, 1); + default: return next_utf8(s, n, i, cp); + } +} + +static void put16(uint8_t *d, uint32_t u, int big) { + d[big ? 0 : 1] = (uint8_t)(u >> 8); + d[big ? 1 : 0] = (uint8_t)u; +} + +/* cp を書く（cp は有効なスカラー値）。容量不足なら CONDE_OUTPUT_EOF。 */ +static int32_t put(uint8_t *d, uint32_t cap, uint32_t *o, uint32_t enc, uint32_t cp) { + uint32_t p = *o, len, k; + if (enc == CODE_UTF8 || enc == CODE_UTF8_BOM) { + len = cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4; + if (cap - p < len) + return CONDE_OUTPUT_EOF; + if (len == 1) { + d[p] = (uint8_t)cp; + } else { + static const uint8_t lead[5] = {0, 0, 0xC0, 0xE0, 0xF0}; + d[p] = (uint8_t)(lead[len] | cp >> 6 * (len - 1)); + for (k = 1; k < len; ++k) + d[p + k] = (uint8_t)(0x80 | (cp >> 6 * (len - 1 - k) & 0x3F)); + } + } else if (enc == CODE_UTF16LE || enc == CODE_UTF16BE) { + int big = enc == CODE_UTF16BE; + len = cp < 0x10000 ? 2 : 4; + if (cap - p < len) + return CONDE_OUTPUT_EOF; + if (len == 2) { + put16(d + p, cp, big); + } else { + cp -= 0x10000u; + put16(d + p, 0xD800 + (cp >> 10), big); + put16(d + p + 2, 0xDC00 + (cp & 0x3FF), big); + } + } else { + int big = enc == CODE_UTF32BE; + len = 4; + if (cap - p < 4) + return CONDE_OUTPUT_EOF; + for (k = 0; k < 4; ++k) + d[p + k] = (uint8_t)(cp >> 8 * (big ? 3 - k : k)); + } + *o = p + len; + return CONDE_OK; +} + +int32_t cpt_code_detect(const uint8_t *s, uint32_t n, uint32_t *bom_size) { + int32_t enc = CODE_UTF8; + uint32_t len = 0; + if (!s && n) + return -1; + if (n >= 4 && s[0] == 0xFF && s[1] == 0xFE && !s[2] && !s[3]) { + enc = CODE_UTF32LE; + len = 4; + } else if (n >= 4 && !s[0] && !s[1] && s[2] == 0xFE && s[3] == 0xFF) { + enc = CODE_UTF32BE; + len = 4; + } else if (n >= 3 && s[0] == 0xEF && s[1] == 0xBB && s[2] == 0xBF) { + enc = CODE_UTF8_BOM; + len = 3; + } else if (n >= 2 && s[0] == 0xFF && s[1] == 0xFE) { + enc = CODE_UTF16LE; + len = 2; + } else if (n >= 2 && s[0] == 0xFE && s[1] == 0xFF) { + enc = CODE_UTF16BE; + len = 2; + } + if (bom_size) + *bom_size = len; + return enc; +} + +/* 入力 1 単位（UTF-8 は 1 byte、UTF-16 は 2 byte、UTF-32 は 4 byte。端数は 1 単位に数える）あたりの + * 最悪出力 byte 数。不正な単位が U+FFFD（UTF-8 で 3 byte）になる場合を含む。 + * UTF-16 入力の対は 2 単位で 4 byte なので、単位あたりでは BMP 1 単位の方が大きい。 */ +static uint32_t bound_for(uint32_t src_size, uint32_t from, uint32_t to) { + uint32_t unit = from == CODE_UTF16LE || from == CODE_UTF16BE ? 2 : from == CODE_UTF32LE || from == CODE_UTF32BE ? 4 : 1; + uint32_t units = src_size / unit + (src_size % unit != 0); + uint32_t per = to == CODE_UTF16LE || to == CODE_UTF16BE ? 2 : to == CODE_UTF32LE || to == CODE_UTF32BE ? 4 : 3; + if (unit == 4 && per < 4) + per = 4; /* U+10000 以上は UTF-8 も UTF-16 も 4 byte */ + if (units > 0xFFFFFFFFu / per) + return 0xFFFFFFFFu; + return units * per; +} + +uint32_t cpt_code_bound(uint32_t src_size, uint32_t from, uint32_t to) { + uint32_t bom, size; + uint64_t total; + if (to > CODE_UTF32BE || from > CODE_UTF32BE) + return 0xFFFFFFFFu; + bom = to == CODE_UTF16LE || to == CODE_UTF16BE ? 2 : to == CODE_UTF32LE || to == CODE_UTF32BE ? 4 : 3; + size = bound_for(src_size, from, to); + total = (uint64_t)size + bom; + return total >= 0xFFFFFFFFu ? 0xFFFFFFFFu : (uint32_t)total; +} + +static int32_t convert(uint8_t *dst, uint32_t *dst_size, + const uint8_t *src, uint32_t src_size, + uint32_t from, uint32_t to, uint32_t flags) { + uint32_t cap, i = 0, o = 0, cp, first = 1; + int32_t status = CONDE_OK; + if (!dst_size) + return CONDE_PARAMETER; + cap = *dst_size; + *dst_size = 0; + if ((src_size && !src) || (cap && !dst) || to > CODE_UTF32BE || + from > CODE_UTF32BE || (flags & ~KNOWN_FLAGS)) + return CONDE_PARAMETER; + if (from == CODE_UTF8_BOM) { + if (src_size >= 3 && src[0] == 0xEF && src[1] == 0xBB && src[2] == 0xBF) + i = 3; + } + if (from == CODE_UTF8_BOM) + from = CODE_UTF8; + if (to == CODE_UTF8_BOM || (flags & CODE_ADD_BOM)) { + status = put(dst, cap, &o, to, BOM); + if (status) + goto fail; + } + while (i < src_size) { + int32_t r = next(from, src, src_size, &i, &cp); + if (r) { + if (!(flags & CODE_REPLACE)) { + status = r; + goto fail; + } + cp = REPLACEMENT; + } + if (first && cp == BOM && !r && (flags & CODE_STRIP_BOM)) { + first = 0; + continue; + } + first = 0; + status = put(dst, cap, &o, to, cp); + if (status) + goto fail; + } + *dst_size = o; + return CONDE_OK; +fail: + return status; +} + +#define WRAP(name, enc) \ + int32_t cpt_code_##name##_to_utf32be(uint8_t *dst, uint32_t *dst_size, const uint8_t *src, \ + uint32_t src_size, uint32_t flags) { \ + return convert(dst, dst_size, src, src_size, enc, CODE_UTF32BE, flags); \ + } \ + int32_t cpt_code_utf32be_to_##name(uint8_t *dst, uint32_t *dst_size, const uint8_t *src, \ + uint32_t src_size, uint32_t flags) { \ + return convert(dst, dst_size, src, src_size, CODE_UTF32BE, enc, flags); \ + } + +WRAP(utf8, CODE_UTF8) +WRAP(utf8bom, CODE_UTF8_BOM) +WRAP(utf16le, CODE_UTF16LE) +WRAP(utf16be, CODE_UTF16BE) +WRAP(utf32le, CODE_UTF32LE) diff --git a/foundation/lib/code/code.exports b/foundation/lib/code/code.exports new file mode 100644 index 0000000..3626376 --- /dev/null +++ b/foundation/lib/code/code.exports @@ -0,0 +1,13 @@ +# WAL ABI 1.0; helpers are not exported. +cpt_code_utf8_to_utf32be fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf32be_to_utf8 fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf8bom_to_utf32be fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf32be_to_utf8bom fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf16le_to_utf32be fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf32be_to_utf16le fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf16be_to_utf32be fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf32be_to_utf16be fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf32le_to_utf32be fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_utf32be_to_utf32le fn(ptr,ptr,ptr,u32,u32)->i32 +cpt_code_bound fn(u32,u32,u32)->u32 +cpt_code_detect fn(ptr,u32,ptr)->i32 diff --git a/foundation/tests/code/app.c b/foundation/tests/code/app.c new file mode 100644 index 0000000..c0ca0b2 --- /dev/null +++ b/foundation/tests/code/app.c @@ -0,0 +1,72 @@ +/* libcode を WAL として import し、UTF-32BE を中心にした変換・BOM・置換・エラーを確認する。 */ +#include +#include + +static uint8_t out[64], mid[64]; + +static int same(const uint8_t *want, uint32_t want_n, uint32_t size) { + return size == want_n && !memcmp(out, want, want_n); +} + +int main(void) { + /* "A", U+00E9, U+3042, U+1F600 */ + static const uint8_t u8[] = {'A', 0xC3, 0xA9, 0xE3, 0x81, 0x82, 0xF0, 0x9F, 0x98, 0x80}; + static const uint8_t u16le[] = {'A', 0, 0xE9, 0, 0x42, 0x30, 0x3D, 0xD8, 0x00, 0xDE}; + static const uint8_t u16be[] = {0, 'A', 0, 0xE9, 0x30, 0x42, 0xD8, 0x3D, 0xDE, 0x00}; + static const uint8_t u32le[] = {'A', 0, 0, 0, 0xE9, 0, 0, 0, 0x42, 0x30, 0, 0, 0x00, 0xF6, 0x01, 0}; + static const uint8_t u32be[] = {0, 0, 0, 'A', 0, 0, 0, 0xE9, 0, 0, 0x30, 0x42, 0, 1, 0xF6, 0x00}; + static const uint8_t u8bom[] = {0xEF, 0xBB, 0xBF, 'A', 0xC3, 0xA9, 0xE3, 0x81, 0x82, 0xF0, 0x9F, 0x98, 0x80}; + static const uint8_t u16lebom[] = {0xFF, 0xFE, 'A', 0}; + uint32_t size = sizeof(out), bom = 9; + + if (cpt_code_utf8_to_utf32be(out, &size, u8, sizeof(u8), 0) || !same(u32be, sizeof(u32be), size)) return 101; + size = sizeof(out); + if (cpt_code_utf16le_to_utf32be(out, &size, u16le, sizeof(u16le), 0) || !same(u32be, sizeof(u32be), size)) return 102; + size = sizeof(out); + if (cpt_code_utf16be_to_utf32be(out, &size, u16be, sizeof(u16be), 0) || !same(u32be, sizeof(u32be), size)) return 103; + size = sizeof(out); + if (cpt_code_utf32le_to_utf32be(out, &size, u32le, sizeof(u32le), 0) || !same(u32be, sizeof(u32be), size)) return 104; + size = sizeof(out); + if (cpt_code_utf32be_to_utf8(out, &size, u32be, sizeof(u32be), 0) || !same(u8, sizeof(u8), size)) return 105; + size = sizeof(out); + if (cpt_code_utf32be_to_utf16le(out, &size, u32be, sizeof(u32be), 0) || !same(u16le, sizeof(u16le), size)) return 106; + size = sizeof(out); + if (cpt_code_utf32be_to_utf16be(out, &size, u32be, sizeof(u32be), 0) || !same(u16be, sizeof(u16be), size)) return 107; + size = sizeof(out); + if (cpt_code_utf32be_to_utf32le(out, &size, u32be, sizeof(u32be), 0) || !same(u32le, sizeof(u32le), size)) return 108; + /* BOM */ + size = sizeof(out); + if (cpt_code_utf32be_to_utf8bom(out, &size, u32be, sizeof(u32be), 0) || !same(u8bom, sizeof(u8bom), size)) return 109; + size = sizeof(out); + if (cpt_code_utf8bom_to_utf32be(out, &size, u8bom, sizeof(u8bom), 0) || !same(u32be, sizeof(u32be), size)) return 110; + size = sizeof(out); + if (cpt_code_utf32be_to_utf16le(out, &size, u32be, 4, CODE_ADD_BOM) || !same(u16lebom, sizeof(u16lebom), size)) return 111; + size = sizeof(out); + if (cpt_code_utf8_to_utf32be(out, &size, u8bom, sizeof(u8bom), CODE_STRIP_BOM) || !same(u32be, sizeof(u32be), size)) return 112; + if (cpt_code_detect(u16lebom, sizeof(u16lebom), &bom) != CODE_UTF16LE || bom != 2) return 113; + /* 経由して UTF-16LE → UTF-8 */ + size = sizeof(mid); + if (cpt_code_utf16le_to_utf32be(mid, &size, u16le, sizeof(u16le), 0)) return 114; + uint32_t mid_n = size; + size = sizeof(out); + if (cpt_code_utf32be_to_utf8(out, &size, mid, mid_n, 0) || !same(u8, sizeof(u8), size)) return 115; + /* 不正入力 */ + static const uint8_t bad[] = {'a', 0xC0, 0xAF, 'b'}; + static const uint8_t bad_fffd[] = {0, 0, 0, 'a', 0, 0, 0xFF, 0xFD, 0, 0, 0xFF, 0xFD, 0, 0, 0, 'b'}; + size = sizeof(out); + if (cpt_code_utf8_to_utf32be(out, &size, bad, sizeof(bad), 0) != CONDE_DATA || size) return 116; + size = sizeof(out); + if (cpt_code_utf8_to_utf32be(out, &size, bad, sizeof(bad), CODE_REPLACE) || !same(bad_fffd, sizeof(bad_fffd), size)) return 117; + size = sizeof(out); + if (cpt_code_utf8_to_utf32be(out, &size, u8, 2, 0) != CONDE_INPUT_EOF || size) return 118; + static const uint8_t lone[] = {0x3D, 0xD8, 'a', 0}; + size = sizeof(out); + if (cpt_code_utf16le_to_utf32be(out, &size, lone, sizeof(lone), 0) != CONDE_DATA || size) return 119; + /* 容量・引数 */ + size = 5; + if (cpt_code_utf8_to_utf32be(out, &size, u8, sizeof(u8), 0) != CONDE_OUTPUT_EOF || size) return 120; + if (cpt_code_bound(sizeof(u8), CODE_UTF8, CODE_UTF32BE) < sizeof(u32be)) return 121; + size = sizeof(out); + if (cpt_code_utf8_to_utf32be(out, &size, u8, 1, 8) != CONDE_PARAMETER) return 122; + return 42; +} diff --git a/foundation/tests/code/kernel.c b/foundation/tests/code/kernel.c new file mode 100644 index 0000000..e6e13bc --- /dev/null +++ b/foundation/tests/code/kernel.c @@ -0,0 +1,20 @@ +#include +#include "images.h" + +static const struct fnd_image libraries[] = {{img_code, img_code_size}}; +static int32_t pid; + +static uint32_t on_tick(uint32_t ticks) { + struct fnd_proc *p = fnd_proc_get((uint32_t)pid); + if (p && p->state == FND_PROC_EXITED) + return p->exit_code == 42 ? 42 : 0x10000u | (uint32_t)p->exit_code; + return ticks > 15000 ? 0x101u : 0; +} + +int main(void) { + pid = fnd_proc_spawn_wax("code-test", FND_PRIO_APP, img_test, img_test_size, libraries, 1, 0, 0); + if (pid < 0) + return 2; + fnd_set_tick_hook(on_tick); + fnd_kernel_run(); +} diff --git a/foundation/tests/code/test_code.py b/foundation/tests/code/test_code.py new file mode 100644 index 0000000..29679bb --- /dev/null +++ b/foundation/tests/code/test_code.py @@ -0,0 +1,269 @@ +"""libcode (conde-style character code conversion) on the host with ctypes, then actual WAL calls in Foundation U mode.""" +import ctypes +import os +from pathlib import Path +import random +import shutil +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +LIB = ROOT / "foundation/lib/code" +CLANG = os.environ.get("CLANG") or shutil.which("clang") +OK, DATA, PARAMETER, INPUT_EOF, OUTPUT_EOF = 0, 1, 5, 6, 7 +UTF8, UTF8_BOM, UTF16LE, UTF16BE, UTF32LE, UTF32BE = 0, 1, 2, 3, 4, 5 +AUTO = "auto" +REPLACE, STRIP_BOM, ADD_BOM = 1, 2, 4 +CODECS = {UTF8: "utf-8", UTF8_BOM: "utf-8", UTF16LE: "utf-16-le", UTF16BE: "utf-16-be", + UTF32LE: "utf-32-le", UTF32BE: "utf-32-be"} +BOMS = {UTF8: b"", UTF8_BOM: b"\xef\xbb\xbf", UTF16LE: b"\xff\xfe", UTF16BE: b"\xfe\xff", + UTF32LE: b"\xff\xfe\0\0", UTF32BE: b"\0\0\xfe\xff"} + + +NAMES = {UTF8: "utf8", UTF8_BOM: "utf8bom", UTF16LE: "utf16le", UTF16BE: "utf16be", UTF32LE: "utf32le", UTF32BE: "utf32be"} +EXPORTS = [f"cpt_code_{n}_to_utf32be" for n in NAMES.values() if n != "utf32be"] + \ + [f"cpt_code_utf32be_to_{n}" for n in NAMES.values() if n != "utf32be"] + ["cpt_code_bound", "cpt_code_detect"] + + +def run(command, cwd=None, timeout=300): + result = subprocess.run(list(map(str, command)), cwd=cwd, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=timeout) + if result.returncode: + raise AssertionError(result.stdout + result.stderr) + return result.stdout + + +def encode(text, enc, bom=False): + return (BOMS[enc] if bom or enc == UTF8_BOM else b"") + text.encode(CODECS[enc]) + + +def random_text(rng, n): + pools = [(0x20, 0x7E), (0x80, 0x7FF), (0x800, 0xD7FF), (0xE000, 0xFFFD), (0x10000, 0x10FFFF)] + return "".join(chr(rng.randint(*rng.choice(pools))) for _ in range(n)) + + +@unittest.skipUnless(CLANG, "Clang required") +class HostTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp = tempfile.TemporaryDirectory(prefix="code-host-") + cls.addClassCleanup(cls.temp.cleanup) + path = Path(cls.temp.name) / ("code.dll" if os.name == "nt" else "code.so") + command = [CLANG, "-shared", "-O2", "-I", ROOT / "foundation/include", LIB / "code.c", "-o", path] + if os.name == "nt": + for symbol in EXPORTS: + command += ["-Xlinker", "/export:" + symbol] + else: + command += ["-fPIC"] + run(command) + cls.lib = ctypes.CDLL(str(path)) + ptr, u32 = ctypes.c_void_p, ctypes.c_uint32 + for name in EXPORTS: + fn = getattr(cls.lib, name) + if name.endswith("bound"): + fn.argtypes, fn.restype = [u32, u32, u32], u32 + elif name.endswith("detect"): + fn.argtypes, fn.restype = [ptr, u32, ptr], ctypes.c_int32 + else: + fn.argtypes, fn.restype = [ptr, ptr, ptr, u32, u32], ctypes.c_int32 + + def call(self, name, data, capacity, flags): + # An odd offset checks that the byte-wise paths tolerate unaligned buffers. + src = ctypes.create_string_buffer(b"\0" + data) + out = ctypes.create_string_buffer(capacity + 1) + n = ctypes.c_uint32(capacity) + status = getattr(self.lib, name)(ctypes.addressof(out) + 1, ctypes.byref(n), + ctypes.addressof(src) + 1, len(data), flags) + self.assertLessEqual(n.value, capacity) + if status: + self.assertEqual(n.value, 0) + return status, out.raw[1:1 + n.value] + + def convert(self, data, src_enc, dst_enc, flags=0, capacity=1 << 16): + """Direct for UTF-32BE on either side, otherwise through UTF-32BE (BOM flags on the matching leg).""" + if src_enc == AUTO: + ptr = ctypes.create_string_buffer(data) + bom = ctypes.c_uint32(0) + src_enc = self.lib.cpt_code_detect(ptr, len(data), ctypes.byref(bom)) + data = data[bom.value:] + if src_enc == dst_enc == UTF32BE: + return OK, data + if dst_enc == UTF32BE: + return self.call(f"cpt_code_{NAMES[src_enc]}_to_utf32be", data, capacity, flags & ~ADD_BOM) \ + if flags & ADD_BOM == 0 else self._add32(data, src_enc, flags, capacity) + if src_enc == UTF32BE: + return self.call(f"cpt_code_utf32be_to_{NAMES[dst_enc]}", data, capacity, flags) + status, mid = self.call(f"cpt_code_{NAMES[src_enc]}_to_utf32be", data, 1 << 16, flags & ~ADD_BOM) + if status: + return status, b"" + return self.call(f"cpt_code_utf32be_to_{NAMES[dst_enc]}", mid, capacity, flags & ~STRIP_BOM) + + def _add32(self, data, src_enc, flags, capacity): + status, out = self.call(f"cpt_code_{NAMES[src_enc]}_to_utf32be", data, capacity, flags & ~ADD_BOM) + return (status, out) if status else (OK, BOMS[UTF32BE] + out) + + def test_all_pairs_round_trip(self): + rng = random.Random(3) + texts = ["", "A", "\x00", "é", "あ", "\U0001F600", "x", "퟿\U0010ffff"] + texts += [random_text(rng, rng.randrange(1, 60)) for _ in range(60)] + encodings = (UTF8, UTF8_BOM, UTF16LE, UTF16BE, UTF32LE, UTF32BE) + for text in texts: + for a in encodings: + for b in encodings: + with self.subTest(text=text, a=a, b=b): + want = encode(text, b) + self.assertEqual(self.convert(encode(text, a), a, b), (OK, want)) + n = self.lib.cpt_code_bound(len(encode(text, a)), a, b) + self.assertGreaterEqual(n, len(want)) + + def test_bom(self): + text = "aあ\U0001F600" + for enc in CODECS: + enc = UTF8_BOM if enc == UTF8 else enc # ADD_BOM on UTF8 writes the same bytes as UTF8_BOM + data = encode(text, enc, bom=True) + self.assertEqual(self.convert(data, AUTO, UTF8), (OK, text.encode())) + self.assertEqual(self.convert(text.encode(), UTF8, enc, ADD_BOM), (OK, data)) + ptr = ctypes.create_string_buffer(data) + bom = ctypes.c_uint32(99) + self.assertEqual(self.lib.cpt_code_detect(ptr, len(data), ctypes.byref(bom)), enc) + self.assertEqual(bom.value, len(BOMS[enc])) + self.assertEqual(self.convert(b"ab", AUTO, UTF16LE), (OK, "ab".encode("utf-16-le"))) + self.assertEqual(self.convert(b"\xef\xbb\xbfab", UTF8_BOM, UTF16BE), (OK, "ab".encode("utf-16-be"))) + self.assertEqual(self.convert(b"ab", UTF8_BOM, UTF8), (OK, b"ab")) + # without a strip flag an explicit encoding keeps U+FEFF as a character + self.assertEqual(self.convert(b"\xef\xbb\xbfab", UTF8, UTF8), (OK, b"\xef\xbb\xbfab")) + self.assertEqual(self.convert(b"\xef\xbb\xbfab", UTF8, UTF8, STRIP_BOM), (OK, b"ab")) + self.assertEqual(self.convert(b"\xff\xfea\0", UTF16LE, UTF8), (OK, b"\xef\xbb\xbfa")) + self.assertEqual(self.convert(b"\xff\xfea\0", UTF16LE, UTF8, STRIP_BOM), (OK, b"a")) + # FF FE 00 00 is UTF-32LE, as documented + self.assertEqual(self.convert(b"\xff\xfe\0\0a\0\0\0", AUTO, UTF8), (OK, b"a")) + self.assertEqual(self.convert(b"", AUTO, UTF8_BOM), (OK, b"\xef\xbb\xbf")) + self.assertEqual(self.convert(b"\xef\xbb\xbfab", UTF8_BOM, UTF32BE), (OK, "ab".encode("utf-32-be"))) + self.assertEqual(self.convert(b"", UTF8, UTF32BE, ADD_BOM), (OK, BOMS[UTF32BE])) + self.assertEqual(self.lib.cpt_code_detect(None, 0, None), UTF8) + self.assertLess(self.lib.cpt_code_detect(None, 1, None), 0) + + def test_utf8_validation(self): + bad = [b"\x80", b"\xc0\xaf", b"\xc1\xbf", b"\xe0\x80\x80", b"\xe0\x9f\xbf", b"\xed\xa0\x80", + b"\xed\xbf\xbf", b"\xf0\x80\x80\x80", b"\xf4\x90\x80\x80", b"\xf5\x80\x80\x80", b"\xff", + b"\xc2a", b"\xe2\x82a", b"\xf0\x9f\x98a"] + good = ["\u0080", "߿", "ࠀ", "퟿", "", "￿", "\U00010000", "\U0010ffff"] + for data in bad: + with self.subTest(data=data): + self.assertEqual(self.convert(data, UTF8, UTF32LE)[0], DATA) + for text in good: + self.assertEqual(self.convert(text.encode(), UTF8, UTF32LE), (OK, text.encode("utf-32-le"))) + for data in (b"\xc2", b"\xe2\x82", b"\xf0\x9f\x98", b"a\xf0"): + self.assertEqual(self.convert(data, UTF8, UTF16LE), (INPUT_EOF, b"")) + + def test_replace_matches_python(self): + rng = random.Random(11) + for enc in (UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE): + for _ in range(600): + width = {UTF8: 1, UTF16LE: 2, UTF16BE: 2, UTF32LE: 4, UTF32BE: 4}[enc] + text = random_text(rng, rng.randrange(0, 6)) + data = bytearray(encode(text, enc)) + for _ in range(rng.randrange(0, 4)): + action = rng.randrange(3) + if action == 0 and data: + data[rng.randrange(len(data))] = rng.choice((0, 0x80, 0xBF, 0xC0, 0xD8, 0xDC, 0xDF, 0xE0, 0xED, 0xF4, 0xFF)) + elif action == 1 and data: + del data[rng.randrange(len(data)):] + else: + data[rng.randrange(len(data) + 1):0] = bytes(rng.randrange(256) for _ in range(rng.randrange(1, width + 2))) + data = bytes(data) + want = data.decode(CODECS[enc], errors="replace") + with self.subTest(enc=enc, data=data): + status, got = self.convert(data, enc, UTF32LE, REPLACE) + self.assertEqual((status, got.decode("utf-32-le")), (OK, want)) + strict = self.convert(data, enc, UTF32LE) + try: + data.decode(CODECS[enc]) + self.assertEqual(strict[0], OK) + except UnicodeDecodeError: + self.assertIn(strict[0], (DATA, INPUT_EOF)) + + def test_utf16_and_utf32_errors(self): + self.assertEqual(self.convert(b"\x3d\xd8", UTF16LE, UTF8), (INPUT_EOF, b"")) # high surrogate alone + self.assertEqual(self.convert(b"\x3d\xd8a\0", UTF16LE, UTF8), (DATA, b"")) + self.assertEqual(self.convert(b"\x00\xde", UTF16LE, UTF8), (DATA, b"")) # low surrogate alone + self.assertEqual(self.convert(b"a\0b", UTF16LE, UTF8), (INPUT_EOF, b"")) # odd length + self.assertEqual(self.convert(b"a\0b", UTF16LE, UTF8, REPLACE), (OK, b"a\xef\xbf\xbd")) + self.assertEqual(self.convert(b"\0\xd8\0\0", UTF32LE, UTF8)[0], DATA) # surrogate + self.assertEqual(self.convert(b"\0\0\x11\0", UTF32LE, UTF8)[0], DATA) # > U+10FFFF + self.assertEqual(self.convert(b"a\0\0", UTF32LE, UTF8), (INPUT_EOF, b"")) + + def test_output_limits(self): + data = "aéあ\U0001F600".encode() + for to in (UTF8_BOM, UTF16LE, UTF32BE): + full = self.convert(data, UTF8, to)[1] + for capacity in range(len(full)): + self.assertEqual(self.convert(data, UTF8, to, capacity=capacity), (OUTPUT_EOF, b"")) + self.assertEqual(self.convert(data, UTF8, to, capacity=len(full)), (OK, full)) + + def test_bound_covers_worst_cases(self): + rng = random.Random(5) + encodings = (UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE) + for _ in range(3000): + a, b = rng.choice(encodings), rng.choice(encodings) + data = bytes(rng.choice((0, 0x41, 0x80, 0xD8, 0xDC, 0xE3, 0xFF)) for _ in range(rng.randrange(0, 20))) + bound = self.lib.cpt_code_bound(len(data), a, b) + status, out = self.convert(data, a, b, REPLACE | ADD_BOM, capacity=bound) + self.assertEqual(status, OK, (data, a, b)) + self.assertLessEqual(len(out), bound) + self.assertEqual(self.lib.cpt_code_bound(0, UTF8, UTF8_BOM), 3) + self.assertEqual(self.lib.cpt_code_bound(0xFFFFFFFF, UTF32LE, UTF8), 0xFFFFFFFF) + self.assertEqual(self.lib.cpt_code_bound(1, 9, UTF8), 0xFFFFFFFF) + + def test_parameters(self): + self.assertEqual(self.convert(b"a", UTF8, UTF32BE, 8)[0], PARAMETER) + self.assertEqual(self.lib.cpt_code_bound(1, 9, UTF8), 0xFFFFFFFF) + self.assertEqual(self.lib.cpt_code_bound(1, UTF8, 9), 0xFFFFFFFF) + self.assertEqual(self.lib.cpt_code_utf8_to_utf32be(None, None, None, 0, 0), PARAMETER) + n = ctypes.c_uint32(0) + self.assertEqual(self.lib.cpt_code_utf8_to_utf32be(None, ctypes.byref(n), None, 1, 0), PARAMETER) + self.assertEqual(self.lib.cpt_code_utf8_to_utf32be(None, ctypes.byref(n), None, 0, 0), OK) + + +@unittest.skipUnless(CLANG, "Clang required") +class GuestTests(unittest.TestCase): + def test_wal_in_foundation(self): + suffix = ".exe" if os.name == "nt" else "" + default_bench = ROOT / ("target/release/src32_testbench" + suffix) + if not default_bench.exists(): + default_bench = ROOT / ("target/debug/src32_testbench" + suffix) + bench = Path(os.environ.get("SRC32_TESTBENCH", default_bench)) + if not bench.exists(): + self.skipTest("src32_testbench required") + with tempfile.TemporaryDirectory(prefix="code-guest-") as directory: + out = Path(directory) + (out / "config.yaml").write_text("", encoding="utf-8") + driver = [sys.executable, ROOT / "tools/llvm-src32/driver.py", "--clang", CLANG] + for opt in ("-O1", "-O2"): + with self.subTest(optimization=opt): + run([sys.executable, LIB / "build.py", out, "--clang", CLANG, "--optimization=" + opt]) + run([*driver, HERE / "app.c", "-I", ROOT / "foundation/include", opt, + "--wax3", "--image-id", "code_test", "--import-wal", out / "libcode.wal", + "--origin", "0xC00000", "--stack-top", "0x1000000", "--user-stack-size", "0x240000", + "-o", out / "test.wax"]) + lines = ["#include "] + for name, path in {"code": out / "libcode.wal", "test": out / "test.wax"}.items(): + data = path.read_bytes() + words = data + bytes(-len(data) % 4) + text = ",".join(hex(int.from_bytes(words[i:i+4], "big")) + "u" for i in range(0, len(words), 4)) + lines += [f"#define img_{name}_size {len(data)}u", f"static const uint32_t img_{name}[] = {{{text}}};"] + (out / "images.h").write_text("\n".join(lines), encoding="utf-8") + run([*driver, HERE / "kernel.c", *(ROOT / "foundation/kernel" / name for name in + ("sched.c", "proc.c", "mm.c", "trap.c", "ipc.c", "fb.c", "sgu.c", "sgc.c", "pec.c", "blk.c", "wax.c")), + "-I", ROOT / "foundation/include", "-I", ROOT / "sdk/cpt32/include", "-I", out, + "-O1", "--origin", "0x10000", "--stack-top", "0x100000", "-o", out / "kernel.wax"]) + for cpu in ("scalar", "pipeline"): + self.assertIn("PASS:", run([bench.resolve(), out / "kernel.wax", "42", "--cpu", cpu, + "--frames", "1200"], cwd=out, timeout=300)) + + +if __name__ == "__main__": + unittest.main() diff --git a/foundation/tests/disk/test_disk_boot.py b/foundation/tests/disk/test_disk_boot.py index 074f6ca..999860b 100644 --- a/foundation/tests/disk/test_disk_boot.py +++ b/foundation/tests/disk/test_disk_boot.py @@ -113,7 +113,7 @@ def test_image_layout(self): self.assertEqual(boot[module['offset']:][:module['size']], image) self.assertEqual([m['caps'] for m in modules], [8, 12, 8, 0]) # SVC for the services, DISK for disk self.assertEqual(len(modules[0]['grants']), 4) # the console's MMIO windows (VDP, text, wide font, input) - for path in ('/bin/shell.wax', '/bin/echo.wax', '/lib/fnd.1.wal', '/lib/lzma.1.wal', '/lib/rle.1.wal'): + for path in ('/bin/shell.wax', '/bin/echo.wax', '/lib/fnd.1.wal', '/lib/lzma.1.wal', '/lib/rle.1.wal', '/lib/code.1.wal'): inode = volume.resolve(path) self.assertEqual(volume.read(inode), (self.build / 'root' / path[1:]).read_bytes()) self.assertEqual(inode['mode'], 0x003D) diff --git a/memory/2026-10-24-issue-160-libcode.md b/memory/2026-10-24-issue-160-libcode.md new file mode 100644 index 0000000..e5d41e2 --- /dev/null +++ b/memory/2026-10-24-issue-160-libcode.md @@ -0,0 +1,7 @@ +# libcode（文字コード変換 WAL） + +- `foundation/lib/code`、image ID `code` 1.0、`/lib/code.1.wal`。API は ``。owner の指示で **UTF-32BE を基準**にし、`cpt_code__to_utf32be`／`cpt_code_utf32be_to_`（`utf8`、`utf8bom`、`utf16le`、`utf16be`、`utf32le`）の対にした。直接の `convert(from,to)` は作らない（他の符号化どうしは呼出側が経由する）。 +- 戻り値・`*dst_size` は conde と同じ。`flags` で `CODE_REPLACE`（U+FFFD、maximal subpart）、`CODE_STRIP_BOM`、`CODE_ADD_BOM`。既定は不正入力でエラー（`CONDE_DATA`、末尾の途中切れは `CONDE_INPUT_EOF`）。 +- BOM の扱い: `utf8bom` は出力で必ず付け、入力で読み飛ばす。明示的な UTF-16/32 の先頭 U+FEFF は文字として写す（Unicode の規定どおり）。`cpt_code_detect` は `FF FE 00 00` を UTF-32LE とする。 +- 落とし穴: llvm-src32 は `llvm.uadd.sat` を持たない。32 bit のオーバーフロー判定を `a + b < a` や飽和風に書くと clang が intrinsic にするので、`uint64_t` で足して比較する。 +- 検証: `CLANG=clang-20 python3 -m unittest foundation.tests.code.test_code`（ホストは Python の codecs と照合、`errors="replace"` の結果とも一致を見る。Foundation 上は `app.c`）。ディスクには `boot/build.py` が入れ、`tests/disk` が存在を確認する。