deps: adopt libcma alpha.10 / libcmt 0.18.0, and fix the riscv64 link so applications can actually use the crate

.github/workflows/test.yml

@@ -17,6 +17,31 @@ jobs:
 
       - uses: dtolnay/rust-toolchain@stable
 
+      # DEPENDENCIES.lock is the human-readable record of which upstream revisions the
+      # submodules are pinned to. It drifted from the actual pins before (it named
+      # machine-guest-tools v0.17.2 while the submodule sat on a later main commit), and a
+      # wrong record is worse than none when the question is "which libcmt is this built
+      # against?". Cheap to assert, so assert it on every push.
+      - name: Verify DEPENDENCIES.lock matches the submodule pins
+        run: |
+          set -euo pipefail
+          status=0
+          while read -r repo tag sha; do
+            [[ -z "${repo:-}" || "${repo:0:1}" == "#" ]] && continue
+            path="third_party/${repo##*/}"
+            actual=$(git ls-tree HEAD "$path" | awk '{print $3}')
+            if [[ -z "$actual" ]]; then
+              echo "::error::DEPENDENCIES.lock names $repo but $path is not a submodule of this tree"
+              status=1
+            elif [[ "$actual" != "$sha" ]]; then
+              echo "::error::$repo ($tag): DEPENDENCIES.lock says $sha, $path is pinned to $actual"
+              status=1
+            else
+              echo "ok: $repo $tag $sha"
+            fi
+          done < DEPENDENCIES.lock
+          exit $status
+
       - name: Install libclang for bindgen
         run: sudo apt-get update && sudo apt-get install -y clang libclang-dev
 
@@ -129,6 +154,128 @@ jobs:
           file "$obj" | grep -F 'RISC-V'
           rm -f "$obj"
 
+      # Inspecting the archive is not enough: libcma.a's parser_impl.o references libcmt's
+      # cmt_abi_*/cmt_buf_* symbols, which live OUTSIDE the archive. Archive members are pulled
+      # lazily, so a missing libcmt only shows up when something actually calls the C parser.
+      # This links a real riscv64 binary that does, against the same `-lcma -lcmt` that build.rs
+      # emits for the riscv64 backend, inside the builder image where the machine-guest-tools
+      # package supplies libcmt. tests/c_parser_link.rs is the host-side counterpart.
+      - name: Link a real riscv64 binary against libcma + libcmt
+        working-directory: third_party/machine-asset-tools
+        run: |
+          set -euo pipefail
+          cat > cmt_link_probe.c <<'PROBE'
+          #include <libcma/parser.h>
+          #include <string.h>
+          int main(void) {
+              unsigned char payload[52];
+              memset(payload, 0, sizeof payload);
+              payload[19] = 1;    /* sender */
+              payload[51] = 4;    /* value  */
+              cmt_rollup_advance_t advance;
+              memset(&advance, 0, sizeof advance);
+              advance.payload.length = sizeof payload;
+              advance.payload.data = payload;
+              cma_parser_input_t out;
+              memset(&out, 0, sizeof out);
+              /* Forces parser_impl.o (and therefore libcmt) into the link. */
+              return cma_parser_decode_advance(
+                  CMA_PARSER_INPUT_TYPE_ETHER_DEPOSIT, &advance, &out) == 0 ? 0 : 1;
+          }
+          PROBE
+          # The probe is written into the working directory rather than /tmp so that the single
+          # existing mount carries it — a second `-v /tmp/...` bind is refused outright by Docker
+          # Desktop, which does not share /tmp by default.
+          trap 'rm -f cmt_link_probe.c' EXIT
+          docker run --platform linux/riscv64 --rm \
+            -v "$PWD":/mnt/app -w /mnt/app libcma-builder \
+            bash -euxc '
+              g++ -std=c++20 -Iinclude -Ithird-party -x c++ cmt_link_probe.c \
+                  -L build/riscv64 -lcma -lcmt -lstdc++ -o /tmp/cmt_link_probe
+              # `readelf`, not `file`: the builder image ships binutils (it has g++) but no
+              # file(1). The neighbouring artifact check can use file(1) because it runs on the
+              # runner rather than inside this container.
+              readelf -h /tmp/cmt_link_probe | grep -F "RISC-V"
+              # Runs natively here (the container IS riscv64), so this also proves the linked
+              # C parser actually decodes the vector rather than merely resolving symbols.
+              /tmp/cmt_link_probe
+            '
+
+
+  # The riscv64 backend as an APPLICATION consumes it: cross-compiled and linked into a
+  # fully static executable. Split out of riscv-link-check deliberately — that job runs a
+  # riscv64 container under QEMU, which is slow, and none of this needs emulation. build.rs
+  # cross-compiles libcma natively with the RISC-V GCC 14 toolchain, so the two jobs run in
+  # parallel and the wall clock is the slower of the two rather than their sum.
+  #
+  # Why an executable: `cargo build` on a library only emits an rlib, whose members are pulled
+  # lazily, so a library build cannot see a missing or wrongly-bound native dependency.
+  # Why static: the Cartesi machine rootfs has no `/lib/ld.so.1`, so an application retaining
+  # any NEEDED entry cannot be exec'd, and the machine reports only
+  # `dapp failed to start with No such file or directory`. A dynamically-bound libstdc++ used
+  # to slip through exactly here.
+  riscv-rust-link:
+    runs-on: ubuntu-latest
+    env:
+      CMA_RISCV64_CXX: riscv64-linux-gnu-g++-14
+      CMA_RISCV64_CC: riscv64-linux-gnu-gcc-14
+      # Must match the version the application runtime image installs, so the app links and
+      # runs against one libcmt ABI. Keep in step with DEPENDENCIES.lock.
+      MACHINE_GUEST_TOOLS_VERSION: "0.18.0"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Install the RISC-V GCC 14 cross toolchain + bindgen deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            g++-14-riscv64-linux-gnu gcc-14-riscv64-linux-gnu binutils-riscv64-linux-gnu \
+            clang libclang-dev build-essential make wget curl
+
+      # Boost is ~210 MB and build.rs fetches it through machine-asset-tools' Makefile. Cache
+      # the download so only the first run on a given Boost version pays for it.
+      - name: Cache third-party downloads
+        uses: actions/cache@v4
+        with:
+          path: third_party/machine-asset-tools/third-party/downloads
+          key: mat-downloads-${{ hashFiles('third_party/machine-asset-tools/Makefile') }}
+
+      - name: Stage riscv64 libcmt into the cross sysroot
+        run: |
+          set -euxo pipefail
+          # Exactly what an application Dockerfile must do — see the README section on the
+          # riscv64 libcmt requirement. libcma.a does not contain libcmt's cmt_abi_*/cmt_buf_*.
+          curl -fsSL -o /tmp/gt.deb \
+            "https://github.com/cartesi/machine-guest-tools/releases/download/v${MACHINE_GUEST_TOOLS_VERSION}/machine-guest-tools_riscv64.deb"
+          dpkg-deb -x /tmp/gt.deb /tmp/gt
+          sudo cp -a /tmp/gt/usr/lib/libcmt.a   /usr/riscv64-linux-gnu/lib/
+          sudo cp -a /tmp/gt/usr/include/libcmt /usr/riscv64-linux-gnu/include/
+
+      - name: Cross-link a fully static riscv64 Rust binary
+        run: |
+          set -euxo pipefail
+          rustup target add riscv64gc-unknown-linux-gnu
+
+          RUSTFLAGS="-C target-feature=+crt-static -C linker=riscv64-linux-gnu-g++-14" \
+            cargo build --release --no-default-features --features riscv64 \
+              --target riscv64gc-unknown-linux-gnu --example link_probe
+
+          bin=target/riscv64gc-unknown-linux-gnu/release/examples/link_probe
+          file "$bin" | grep -F 'RISC-V'
+          if riscv64-linux-gnu-readelf -l "$bin" | grep -i 'interpreter'; then
+            echo "::error::riscv64 binary requests an ELF interpreter; the machine rootfs has no /lib/ld.so.1"
+            exit 1
+          fi
+          if riscv64-linux-gnu-readelf -d "$bin" 2>/dev/null | grep 'NEEDED'; then
+            echo "::error::riscv64 binary has dynamic dependencies; it must be fully static"
+            exit 1
+          fi
+          echo "ok: fully static riscv64 executable, no interpreter and no NEEDED entries"
+
   # ===========================================================================
   # TODO(cross-arch-differential): the ULTIMATE host<->machine reproducibility
   # invariant — not yet implemented (heavy: needs a riscv64 build AND running it

CHANGELOG.md

@@ -19,10 +19,31 @@ While the crate is pre-1.0 (`0.0.x`), any release may contain breaking changes.
   (MSRV `1.74`), a `documentation` link, and `[package.metadata.docs.rs]` (an
   offline `mock`-only docs build). Added `CHANGELOG.md`, `CONTRIBUTING.md`,
   `SECURITY.md`, `deny.toml`, `rust-toolchain.toml`, and `rustfmt.toml`.
+- **`tests/c_parser_link.rs`** — forces libcma's `parser_impl.o` into the link so
+  the libcmt dependency below cannot silently regress, and checks the C parser
+  agrees with the Rust port on the canonical ether-deposit vector. The
+  `riscv-link-check` CI job gained the equivalent guard: it now links a real
+  riscv64 binary that calls the C parser instead of only `ar`-inspecting the
+  archive.
+- CI now asserts `DEPENDENCIES.lock` matches the actual submodule pins.
 
 ### Changed
+- **Vendored `libcma` bumped to `v0.1.0-alpha.10` and `libcmt` to `v0.18.0`.**
+  libcma's own API and implementation are unchanged between `alpha.9` and
+  `alpha.10` — the release only bumps the libcmt it builds against (0.17.2 →
+  0.18.0), adds `ioctl.h` to the extracted libcmt header set, and makes
+  `make docker-image` pass `MACHINE_GUEST_TOOLS_VERSION` as a build-arg so the
+  image's libcmt can no longer drift from the staged headers. The libcmt changes
+  that reach bindgen are renames only, with no size, alignment, or field-offset
+  change: `cmt_rollup_t.finish_root_hash` → `finish_outputs_merkle_root`, struct
+  tag `cmt_rollup_finish` → `cmt_rollup_finish_s` (the `_t` typedef is
+  unchanged), and `HTIF_YIELD_REASON_{ADVANCE,INSPECT}` →
+  `..._{ADVANCE,INSPECT}_STATE`. No source change was needed in this crate.
+  Note the Cartesi SDK 12 application templates also ship guest-tools 0.18.0, so
+  this keeps the crate aligned with the runtime its apps execute in.
 - **Vendored `libcma` bumped to the uint96 single-asset format (drive format v2;
-  machine-asset-tools `e4bfc24`).** The single-asset drive record widened its
+  machine-asset-tools `v0.1.0-alpha.9`, `19a1e5e` — the merge of the `e4bfc24`
+  feature branch).** The single-asset drive record widened its
   balance from `uint64` to `uint96`, consuming the former 4-byte pad: the 32-byte
   record is now `balance_lo (u64 LE) | balance_hi (u32 LE) | owner (20B)`, with the
   owner moved from offset 8 to **offset 12**. Total supply and virtual (internal
@@ -46,12 +67,54 @@ While the crate is pre-1.0 (`0.0.x`), any release may contain breaking changes.
 - **Relocation safety for `cma_ledger_t`.** The self-referential C++ ledger is
   now boxed so it is not moved after construction, preventing dangling internal
   self-pointers.
+- **`libcmt` is now linked alongside `libcma` on the real backends.** `libcma.a`
+  archives only libcma's own objects; its `parser_impl.o` references libcmt's C
+  ABI helpers (`cmt_abi_*`, `cmt_buf_*`), which are not in the archive, but
+  `build.rs` emitted only `-lcma -lstdc++`. Because archive members are pulled
+  lazily and nothing in this crate calls the C parser (`parser.rs` is pure Rust
+  over `alloy`), the gap was invisible here and only broke DOWNSTREAM consumers
+  that called `cma_parser_decode_advance`/`_inspect` or `cma_parser_encode_voucher`
+  — with a dozen `undefined symbol: cmt_abi_*` at link time. `host-real` now
+  builds and statically links `libcmt.a`; `riscv64` emits a plain `-lcmt`,
+  resolved by the machine-guest-tools package in the application image (upstream
+  deliberately does not build libcmt for riscv64, as its real io backend needs
+  the cartesi kernel headers).
+- **Stale prebuilt archives are no longer reused across submodule bumps.**
+  `build.rs` cached the C++ build on the mere existence of `libcma.a`, so a bump
+  of `machine-asset-tools` silently linked the previously-compiled archive
+  against the new headers. The object directory is now stamped with the inputs
+  that determine it (both submodule revisions, target arch, compiler overrides)
+  and rebuilt from scratch when they change.
+- **`+crt-static` targets now link the C++ runtime statically too.** The Cartesi
+  Rust application template sets `-C target-feature=+crt-static`, which IS
+  honoured for `riscv64gc-unknown-linux-gnu`: glibc links in statically. But
+  `build.rs` requested libstdc++ as a *dylib*, leaving the application with a
+  static libc, a lone `NEEDED libstdc++.so.6`, and therefore an interpreter of
+  `/lib/ld.so.1` — a path that does not exist in the machine rootfs (Ubuntu
+  riscv64 ships the loader as `/lib/ld-linux-riscv64-lp64d.so.1`). The machine
+  could not exec the application at all, reporting only `dapp failed to start
+  with No such file or directory`, which names neither the loader nor libstdc++.
+  Under `+crt-static`, libstdc++ and libcmt are now bound statically, with each
+  archive located via `<compiler> -print-file-name=` so the paths come from the
+  toolchain. Verified end to end by booting a `cartesi create --template rust`
+  application on a real Cartesi machine.
+- **A partially-staged `third-party/libcmt` is now repaired automatically.**
+  Upstream's staging target is a directory, whose mtime is always newer than the
+  tarball it came from, so make treats an incomplete copy as up to date forever
+  and the libcmt compile dies on a missing `src/buf.c`.
 
 ### Security / hardening
 - Mutual-exclusion guards: enabling more than one backend feature now fails at
   compile time (`compile_error!`) instead of silently letting the mock win in a
   real build.
-- `build.rs` verifies a checksum of the vendored/built libcma source.
+- `build.rs` verifies the nlohmann/json header it downloads against a SHA-256
+  pinned in that file — it is the one build input fetched over the network
+  rather than pinned as a submodule. (Earlier wording here claimed a checksum
+  over "the vendored/built libcma source"; libcma is pinned by submodule commit
+  SHA, not checksummed.)
+- `SECURITY.md` documents why the C/C++ dependencies are pinned by commit SHA
+  rather than tag name: upstream tags are mutable, and machine-asset-tools'
+  `v0.1.0-alpha.8` has already been force-moved once.
 
 ### Changed (breaking)
 - **Migrated off the EOL `ethers-rs` onto `alloy`** (`alloy-primitives` +

DEPENDENCIES.lock

@@ -1 +1,2 @@
-cartesi/machine-guest-tools v0.17.2 3d838a2f80cd8fa614e38ea93efcfabe8ec65bea
+Mugen-Builders/machine-asset-tools v0.1.0-alpha.10 25d32cd895031b0e22015dbc5b91ba8821762830
+cartesi/machine-guest-tools v0.18.0 5222250c69371f7cbe96e6082699a8ca22b5969c

README.md

@@ -80,6 +80,71 @@ If you forget `default-features = false`, enabling `host-real` or `riscv64`
 alongside the default `mock` trips the mutual-exclusivity `compile_error!` — read
 its message; the fix is to disable default features.
 
+### `riscv64`: your application must supply `libcmt`
+
+`libcma.a` contains only libcma's own objects. Its parser references libcmt's C
+ABI helpers (`cmt_abi_*`, `cmt_buf_*`), so `build.rs` emits `-lcmt` for the
+`riscv64` backend and **`libcmt.a` must be on the linker's search path in the
+stage that cross-links your binary**. Upstream deliberately does not build libcmt
+for riscv64 — its real io backend needs the Cartesi Linux kernel headers, which
+exist only inside the guest — so it comes from the released `machine-guest-tools`
+package. (`host-real` needs nothing extra: `build.rs` builds and statically links
+`libcmt.a` itself, from the vendored sources with the mock io backend.)
+
+Installing machine-guest-tools in your application's *runtime* stage is **not
+enough** — by then the binary is already linked. Stage it into the cross sysroot
+of the build stage, using the same guest-tools version your runtime stage
+installs, so you link and run against one libcmt ABI:
+
+```dockerfile
+# in the cross-build stage, before `cargo build`
+ARG MACHINE_GUEST_TOOLS_VERSION
+ADD https://github.com/cartesi/machine-guest-tools/releases/download/v${MACHINE_GUEST_TOOLS_VERSION}/machine-guest-tools_riscv64.deb /tmp/gt.deb
+RUN dpkg-deb -x /tmp/gt.deb /tmp/gt && \
+    cp -a /tmp/gt/usr/lib/libcmt.a     /usr/riscv64-linux-gnu/lib/ && \
+    cp -a /tmp/gt/usr/include/libcmt   /usr/riscv64-linux-gnu/include/
+
+# libcma is C++20/23, so it needs GCC >= 14; the stock template installs 13.
+ENV CMA_RISCV64_CXX=riscv64-linux-gnu-g++-14 \
+    CMA_RISCV64_CC=riscv64-linux-gnu-gcc-14
+```
+
+Symptom if this is missing: the C++ compile succeeds and the failure appears only
+at the final link, as `undefined symbol: cmt_abi_get_uint`, `cmt_buf_split`, and
+friends. It surfaces only once something calls the C parser bindings, because
+static archive members are pulled lazily.
+
+#### `+crt-static` and the C++ runtime
+
+The Cartesi Rust template builds with `-C target-feature=+crt-static`, and that
+flag **is** honoured for `riscv64gc-unknown-linux-gnu` — glibc is linked in
+statically. A pure-Rust application therefore produces a fully static binary with
+no `PT_INTERP` at all, which is what the machine's rootfs expects.
+
+Pulling in a C++ library changes that. If libstdc++ were linked dynamically, the
+binary would keep a single `NEEDED libstdc++.so.6` and, with it, an interpreter
+of `/lib/ld.so.1` — a path that does **not** exist in the machine rootfs (Ubuntu
+riscv64 installs the loader as `/lib/ld-linux-riscv64-lp64d.so.1`). The machine
+then cannot exec the application and reports only:
+
+```
+WARN rollup_http_server::dapp_process] throwing exception because dapp failed to
+     start with No such file or directory (os error 2)
+```
+
+which names neither the loader nor libstdc++.
+
+So when `+crt-static` is set, `build.rs` binds libstdc++ and libcmt statically as
+well, locating each archive through `<compiler> -print-file-name=` so the paths
+come from the toolchain rather than being guessed. Nothing is required of the
+application beyond providing a C++ toolchain that ships `libstdc++.a` — the
+`g++-14-riscv64-linux-gnu` package does. Verify with:
+
+```console
+$ riscv64-linux-gnu-readelf -l dapp | grep -i interpreter   # expect no output
+$ riscv64-linux-gnu-readelf -d dapp | grep NEEDED           # expect no output
+```
+
 ### Determinism / reproducibility
 
 `host-real` and `riscv64` compile the C++ `libcma` with SIMD-free / generic flags

SECURITY.md

@@ -23,6 +23,20 @@ Please include a description, the affected version(s), and reproduction steps
 where possible. You can expect an initial acknowledgement within a reasonable
 time frame; we will then coordinate a fix and a disclosure timeline with you.
 
+## Vendored C/C++ dependencies
+
+The real backends compile libcma (and, on the host, libcmt) from the sources in
+`third_party/`, which are git submodules pinned by **commit SHA** — never by tag
+name. This is deliberate: upstream tags are mutable. `v0.1.0-alpha.8` of
+machine-asset-tools has already been force-moved once, and a fetch of a
+re-pointed tag would silently change what gets compiled if we tracked tag names.
+`DEPENDENCIES.lock` records the pinned revisions in readable form and CI asserts
+it matches the actual submodule pins.
+
+The one dependency fetched over the network at build time rather than pinned as
+a submodule is the nlohmann/json single header, so `build.rs` verifies it
+against a SHA-256 pinned in that file. A mismatch fails the build.
+
 ## Scope note
 
 The default `mock` backend is an in-memory stub for development and testing — it