From 1c92013a390d3b6672480fedc38e790c67182288 Mon Sep 17 00:00:00 2001 From: Codex Mozc Autonomy Date: Sat, 14 Nov 2026 07:58:09 -0500 Subject: [PATCH 1/2] =?UTF-8?q?Lite=20APK=20=E3=82=92=E8=BB=BD=E9=87=8F?= =?UTF-8?q?=E5=8C=96=E3=81=97=20F-Droid=20=E6=A4=9C=E8=A8=BC=E3=82=92?= =?UTF-8?q?=E5=BC=B7=E5=8C=96=20/=20Reduce=20Lite=20size=20and=20verify=20?= =?UTF-8?q?F-Droid=20builds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/android.yml | 46 ++++++- app/build.gradle | 48 ++++--- .../compact/CompactEnglishDictionary.kt | 41 ++++-- .../BundledEnglishDictionaryReaderTest.kt | 86 ++++++++++++ docs/fdroid-release.md | 128 ++++++++++-------- tools/release/test_verify_lite_apk.py | 60 ++++++++ tools/release/verify-lite-apk.py | 55 ++++++++ 7 files changed, 380 insertions(+), 84 deletions(-) create mode 100644 app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/BundledEnglishDictionaryReaderTest.kt create mode 100644 tools/release/test_verify_lite_apk.py create mode 100644 tools/release/verify-lite-apk.py diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 600068034..b9fba86ea 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -2,6 +2,20 @@ name: Android Release CI on: workflow_dispatch: + pull_request: + paths: + - '.github/workflows/android.yml' + - 'build.gradle' + - 'settings.gradle' + - 'gradle.properties' + - 'gradle/**' + - 'app/build.gradle' + - 'app/src/main/assets/**' + - 'app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/**' + - 'app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/**' + - 'app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/**' + - 'tools/release/**' + - 'tools/classified-dictionaries/**' push: tags: - 'v*' @@ -15,6 +29,7 @@ concurrency: jobs: build: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest env: ZENZ_MODEL_CACHE_DIR: ${{ github.workspace }}/.zenz-model-cache @@ -92,6 +107,7 @@ jobs: --stacktrace \ --no-daemon \ --max-workers=1 + python3 tools/release/verify-lite-apk.py app/build/outputs/apk/liteStandard/release/*.apk - name: Build and verify Play bundle native symbols env: @@ -147,7 +163,7 @@ jobs: run: sdkmanager "platforms;android-37.0" "build-tools;36.0.0" "ndk;29.0.14206865" - name: Verify release source guard - run: python3 -m unittest discover -s tools/release -p 'test_verify_release_source.py' + run: python3 -m unittest discover -s tools/release - name: Build unsigned F-Droid APK without restored caches env: @@ -161,6 +177,16 @@ jobs: --stacktrace --no-daemon --max-workers=1 python3 tools/release/verify-release-source.py --revision "$GITHUB_SHA" \ app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk + python3 tools/release/verify-lite-apk.py \ + app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk + + - name: Test English dictionary decoding + env: + GRADLE_USER_HOME: ${{ runner.temp }}/fdroid-gradle-${{ matrix.build }} + run: | + ./gradlew :app:testLiteFdroidDebugUnitTest \ + --tests '*BundledEnglishDictionaryReaderTest' \ + --no-daemon --max-workers=1 - name: Upload unsigned F-Droid APK for comparison uses: actions/upload-artifact@v4 @@ -169,8 +195,24 @@ jobs: if-no-files-found: error path: app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk + verify-fdroid: + needs: fdroid + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + name: fdroid-unsigned-first-${{ github.sha }} + path: fdroid/first + - uses: actions/download-artifact@v4 + with: + name: fdroid-unsigned-second-${{ github.sha }} + path: fdroid/second + - name: Require identical clean F-Droid builds + run: cmp fdroid/first/app-liteFdroid-release.apk fdroid/second/app-liteFdroid-release.apk + publish: - needs: [build, fdroid] + if: github.event_name != 'pull_request' + needs: [build, fdroid, verify-fdroid] runs-on: ubuntu-latest steps: - name: Checkout verification tools from the workflow commit diff --git a/app/build.gradle b/app/build.gradle index 7c34dfc9b..608ebd8b9 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -151,9 +151,10 @@ android { } androidResources { - // These assets are already compact binary images. Keeping them stored lets - // Android read them directly instead of inflating them on the IME's cold path. - noCompress += ['kdict', 'compact', 'index', 'erol'] + // Keep only memory-mapped assets stored. The connection matrix is read + // through streams, so its compact data and index can still be compressed. + noCompress += ['kdict', 'reading.compact', 'word.compact', 'token.compact', + 'english.lookup.index', 'erol'] } sourceSets { @@ -213,9 +214,6 @@ abstract class GenerateDictionaryLicenseAssets extends DefaultTask { } abstract class PackageRuntimeDictionaryAssets extends DefaultTask { - @Input - abstract Property getStandardChannel() - @InputDirectory @PathSensitive(PathSensitivity.RELATIVE) abstract DirectoryProperty getMergedAssets() @@ -232,9 +230,7 @@ abstract class PackageRuntimeDictionaryAssets extends DefaultTask { from(mergedAssets) into(packagedAssets) exclude('english/reading.dat.zip', 'english/word.dat', 'english/token.dat.zip') - if (standardChannel.get()) { - exclude('system/tango.dat.zip', 'system/yomi.dat.zip', 'system/token.dat.zip') - } + exclude('system/tango.dat.zip', 'system/yomi.dat.zip', 'system/token.dat.zip') } } } @@ -279,9 +275,7 @@ androidComponents { } variant.sources.assets.addGeneratedSourceDirectory(licenses) { it.outputDirectory } def compactAssets = tasks.register("package${variant.name.capitalize()}RuntimeDictionaryAssets", - PackageRuntimeDictionaryAssets) { - standardChannel.set(variant.productFlavors.any { it.first == 'channel' && it.second == 'standard' }) - } + PackageRuntimeDictionaryAssets) variant.artifacts.use(compactAssets) .wiredWithDirectories({ it.mergedAssets }, { it.packagedAssets }) .toTransform(SingleArtifact.ASSETS.INSTANCE) @@ -757,19 +751,41 @@ tasks.register("generateCompactSystemDictionary") { write(output) } } - writeEntries("reading.compact", 0x4B455231) { output -> // KER1 + // English tries use a small alphabet. Store one-byte alphabet indices, + // then restore the same CharArray at load time. Larger alphabets keep UTF-16. + def writeLabels = { DataOutputStream output, char[] labels -> + char[] alphabet = new String(labels).chars().distinct().sorted().toArray() + .collect { (char) it } as char[] + if (alphabet.length == 0 || alphabet.length > 256) { + output.writeInt(0) + writeArray(output, labels.length, Character.BYTES) { it.asCharBuffer().put(labels) } + } else { + writeArray(output, alphabet.length, Character.BYTES) { it.asCharBuffer().put(alphabet) } + byte[] indexByChar = new byte[Character.MAX_VALUE + 1] + for (int index = 0; index < alphabet.length; index++) { + indexByChar[(int) alphabet[index]] = (byte) index + } + byte[] indices = new byte[labels.length] + for (int index = 0; index < labels.length; index++) { + indices[index] = indexByChar[(int) labels[index]] + } + output.writeInt(indices.length) + output.write(indices) + } + } + writeEntries("reading.compact", 0x4B455232) { output -> // KER2 writeBitSet(output, englishReading[0] as BitSet) writeBitSet(output, englishReading[1] as BitSet) char[] labels = englishReading[2] as char[] int[] termIds = englishReading[3] as int[] - writeArray(output, labels.length, Character.BYTES) { it.asCharBuffer().put(labels) } + writeLabels(output, labels) writeArray(output, termIds.length, Integer.BYTES) { it.asIntBuffer().put(termIds) } } - writeEntries("word.compact", 0x4B455731) { output -> // KEW1 + writeEntries("word.compact", 0x4B455732) { output -> // KEW2 writeBitSet(output, englishWord[0] as BitSet) writeBitSet(output, englishWord[1] as BitSet) char[] labels = englishWord[2] as char[] - writeArray(output, labels.length, Character.BYTES) { it.asCharBuffer().put(labels) } + writeLabels(output, labels) } writeEntries("token.compact", 0x4B455431) { output -> // KET1 short[] costs = englishToken[0] as short[] diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt index 2812ca035..28e48651d 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt @@ -67,39 +67,48 @@ internal object BundledEnglishDictionaryReader { if (size() == (length() + Long.SIZE_BITS - 1) / Long.SIZE_BITS * Long.SIZE_BITS) this else BitSet.valueOf(toLongArray()) - fun readReading(descriptor: AssetFileDescriptor): LOUDSWithTermId = read(descriptor, 0x4B455231) { + fun readReading(descriptor: AssetFileDescriptor): LOUDSWithTermId = readReading(map(descriptor)) + + internal fun readReading(source: ByteBuffer): LOUDSWithTermId = read(source, 0x4B455232) { val lbs = readBitSet() val leaves = readBitSet() - LOUDSWithTermId(lbs, readCharArray(), leaves, readIntArray()) + LOUDSWithTermId(lbs, readLabels(), leaves, readIntArray()) } - fun readWord(descriptor: AssetFileDescriptor): LOUDS = read(descriptor, 0x4B455731) { + fun readWord(descriptor: AssetFileDescriptor): LOUDS = readWord(map(descriptor)) + + internal fun readWord(source: ByteBuffer): LOUDS = read(source, 0x4B455732) { val lbs = readBitSet() val leaves = readBitSet() - LOUDS(lbs, readCharArray(), leaves) + LOUDS(lbs, readLabels(), leaves) } - fun readToken(descriptor: AssetFileDescriptor): TokenArray = read(descriptor, 0x4B455431) { + fun readToken(descriptor: AssetFileDescriptor): TokenArray = read(map(descriptor), 0x4B455431) { TokenArray.fromArrays(readShortArray(), readIntArray(), readBitSet()) } private inline fun read( - descriptor: AssetFileDescriptor, + source: ByteBuffer, magic: Int, decode: Cursor.() -> T, ): T { - val input = Cursor(descriptor) + val input = Cursor(source) require(input.readInt() == magic) { "Invalid bundled English dictionary header" } return input.decode().also { require(input.remaining == 0) { "Unexpected trailing English dictionary data" } } } - private class Cursor(descriptor: AssetFileDescriptor) { - private val buffer: ByteBuffer = FileInputStream(descriptor.fileDescriptor).channel.use { channel -> + private fun map(descriptor: AssetFileDescriptor): ByteBuffer = + FileInputStream(descriptor.fileDescriptor).channel.use { channel -> require(descriptor.length > 0L) { "English dictionary asset must have a known length" } channel.map(FileChannel.MapMode.READ_ONLY, descriptor.startOffset, descriptor.length) - }.order(ByteOrder.BIG_ENDIAN) + } + + private class Cursor(source: ByteBuffer) { + constructor(descriptor: AssetFileDescriptor) : this(map(descriptor)) + + private val buffer = source.duplicate().order(ByteOrder.BIG_ENDIAN) val remaining: Int get() = buffer.remaining() @@ -130,6 +139,18 @@ internal object BundledEnglishDictionaryReader { return CharArray(size).also { take(size * Char.SIZE_BYTES).asCharBuffer().get(it) } } + fun readLabels(): CharArray { + val alphabet = readCharArray() + require(alphabet.size <= 256) { "Invalid English label alphabet" } + if (alphabet.isEmpty()) return readCharArray() + val size = readSizedCount(Byte.SIZE_BYTES) + return CharArray(size) { + val index = buffer.get().toInt() and 0xff + require(index < alphabet.size) { "Invalid English label index: $index" } + alphabet[index] + } + } + fun readShortArray(): ShortArray { val size = readSizedCount(Short.SIZE_BYTES) return ShortArray(size).also { take(size * Short.SIZE_BYTES).asShortBuffer().get(it) } diff --git a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/BundledEnglishDictionaryReaderTest.kt b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/BundledEnglishDictionaryReaderTest.kt new file mode 100644 index 000000000..a6d4d6bcd --- /dev/null +++ b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/BundledEnglishDictionaryReaderTest.kt @@ -0,0 +1,86 @@ +package com.kazumaproject.markdownhelperkeyboard.converter.compact + +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream +import java.nio.ByteBuffer +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Test + +class BundledEnglishDictionaryReaderTest { + @Test + fun readingRestoresLabelsAndTermIds() { + val labels = charArrayOf(' ', 'a', '\u0000', '\uffff', '\ud800') + val source = fixture(0x4B455232, labels) { + writeInt(2) + writeInt(7) + writeInt(42) + } + val reading = BundledEnglishDictionaryReader.readReading(source) + assertArrayEquals(labels, reading.labels) + assertArrayEquals(intArrayOf(7, 42), reading.termIdsSaved) + assertEquals(0, source.position()) + } + + @Test + fun wordSupportsAll256UnsignedAlphabetIndices() { + val labels = CharArray(256) { it.toChar() }.reversedArray() + val word = BundledEnglishDictionaryReader.readWord(fixture(0x4B455732, labels)) + assertArrayEquals(labels, word.labels) + } + + @Test + fun largerAlphabetsAndEmptyLabelsUseRawCharacters() { + for (labels in listOf(CharArray(257) { it.toChar() }, charArrayOf())) { + assertArrayEquals(labels, + BundledEnglishDictionaryReader.readWord(fixture(0x4B455732, labels)).labels) + } + } + + @Test + fun rejectsAnIndexOutsideTheAlphabet() { + val source = fixture(0x4B455732, charArrayOf('a')) + source.put(source.limit() - 1, 1.toByte()) + assertThrows(IllegalArgumentException::class.java) { + BundledEnglishDictionaryReader.readWord(source) + } + } + + @Test + fun rejectsTruncatedLabelsAndTrailingBytes() { + val source = fixture(0x4B455732, charArrayOf('a', 'b')) + assertThrows(IllegalArgumentException::class.java) { + BundledEnglishDictionaryReader.readWord(source.duplicate().apply { limit(limit() - 1) }) + } + val trailing = ByteBuffer.wrap(source.array() + byteArrayOf(0)) + assertThrows(IllegalArgumentException::class.java) { + BundledEnglishDictionaryReader.readWord(trailing) + } + } + + private fun fixture( + magic: Int, + labels: CharArray, + tail: DataOutputStream.() -> Unit = {}, + ): ByteBuffer { + val bytes = ByteArrayOutputStream() + DataOutputStream(bytes).use { out -> + out.writeInt(magic) + repeat(2) { out.writeInt(0) } // Empty LBS and leaf bitsets. + val alphabet = labels.distinct().sorted() + if (alphabet.isEmpty() || alphabet.size > 256) { + out.writeInt(0) + out.writeInt(labels.size) + labels.forEach { out.writeChar(it.code) } + } else { + out.writeInt(alphabet.size) + alphabet.forEach { out.writeChar(it.code) } + out.writeInt(labels.size) + labels.forEach { out.writeByte(alphabet.indexOf(it)) } + } + out.tail() + } + return ByteBuffer.wrap(bytes.toByteArray()) + } +} diff --git a/docs/fdroid-release.md b/docs/fdroid-release.md index d1b15fd77..f2f9018b6 100644 --- a/docs/fdroid-release.md +++ b/docs/fdroid-release.md @@ -1,80 +1,96 @@ -# F-Droid Release Notes +# F-Droid releases -This repository now includes the upstream metadata and build behavior needed to package the `liteFdroid` variant for F-Droid. +The F-Droid package is `com.kazumaproject.markdownhelperkeyboard.lite.fdroid` +(`liteFdroid`). It excludes the optional Zenz and local AI runtimes and models. +Its version is taken from `app/build.gradle`, with the `-lite-fdroid` suffix. -## What ships on F-Droid +## Publication and official availability -- Package name: `com.kazumaproject.markdownhelperkeyboard.lite.fdroid` -- Version name: `1.7.27` -- Version code: `721` -- Gradle flavor: `liteFdroid` +The [GitHub release workflow](../.github/workflows/android.yml) publishes +`app-liteFdroid-release.apk` for a new `v*` tag only after it: -The F-Droid flavor is the lightweight edition. It does not include the optional `zenz` or `gemma` model features. +1. Builds the exact workflow commit twice, on separate runners with empty Gradle + user homes and no restored build or configuration caches. +2. Checks each APK's embedded source revision and dictionary packaging. +3. Requires the unsigned APKs to be byte-for-byte identical. +4. Signs the verified APK, preserving alignment, and verifies signature copying + against the independent unsigned rebuild with `apksigcopier`. +5. Checks that the public tag still resolves to the workflow commit. -## Upstream metadata +Relevant pull requests also run the unsigned F-Droid builds, dictionary decoder +tests, and byte comparison. Signing and publication run on tag pushes or manual +workflow dispatches; manual dispatch does not create a GitHub Release. -F-Droid can read app-store metadata directly from this repository. +A successful GitHub workflow does not establish that the official F-Droid +repository has accepted or distributed that version. Check these separately: -- Text metadata: `fastlane/metadata/android/` -- English locale: `fastlane/metadata/android/en-US/` -- Japanese locale: `fastlane/metadata/android/ja-JP/` +- [Official package listing](https://f-droid.org/packages/com.kazumaproject.markdownhelperkeyboard.lite.fdroid/) +- [Official package version API](https://f-droid.org/api/v1/packages/com.kazumaproject.markdownhelperkeyboard.lite.fdroid) +- [Current fdroiddata metadata](https://gitlab.com/fdroid/fdroiddata/-/blob/master/metadata/com.kazumaproject.markdownhelperkeyboard.lite.fdroid.yml) +- [Upstream reproducibility issue](https://github.com/KazumaProject/JapaneseKeyboard/issues/1293) -The screenshots and icon used for the listing are stored under the `en-US/images/` subtree, which is supported by F-Droid's fastlane-compatible metadata import. +The file under this repository's `fdroid/metadata/` is the historical submission +draft. The app is already in fdroiddata; use its current metadata when validating +an update, rather than replacing it with the old draft. -## Local build command +Never move an existing release tag or replace published APK assets to deliver +a fix. Publish a new version and tag so F-Droid rebuilds the same source as the +reference binary. -Build the exact release APK that matches the F-Droid submission draft with: +## Rebuild a release -```bash -./gradlew assembleLiteFdroidRelease -``` - -If your shell points `JAVA_HOME` to a stale JDK path, run: +Use a fresh clone of the exact public tag. Install the SDK platform, build tools, +and NDK listed in the release workflow. The Gradle wrapper and +`gradle/gradle-daemon-jvm.properties` select the Gradle and daemon JDK versions. +Keep signing properties out of this checkout when building the unsigned APK. ```bash -JAVA_HOME=$(/usr/libexec/java_home -v 17) ./gradlew assembleLiteFdroidRelease +git clone https://github.com/KazumaProject/JapaneseKeyboard.git +cd JapaneseKeyboard +git checkout --detach v +export GRADLE_USER_HOME="$(mktemp -d)" +./gradlew clean :app:assembleLiteFdroidRelease \ + --no-build-cache --no-configuration-cache --no-daemon --max-workers=1 +python3 tools/release/verify-release-source.py \ + --revision "$(git rev-parse HEAD)" --tag v \ + app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk +python3 tools/release/verify-lite-apk.py \ + app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk ``` -The APK is generated at: - -```text -app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk -``` - -## fdroiddata draft - -A starter metadata file for submitting this app to the main F-Droid repository is included at: - -```text -fdroid/metadata/com.kazumaproject.markdownhelperkeyboard.lite.fdroid.yml -``` - -Copy that file into your `fdroiddata/metadata/` checkout, then run the standard validation flow: +Compare the release APK with this unsigned rebuild using the pinned +`apksigcopier` version and signing tools from the workflow: ```bash -fdroid readmeta -fdroid lint com.kazumaproject.markdownhelperkeyboard.lite.fdroid -fdroid checkupdates --allow-dirty com.kazumaproject.markdownhelperkeyboard.lite.fdroid -fdroid build com.kazumaproject.markdownhelperkeyboard.lite.fdroid +apksigcopier compare --unsigned /path/to/published/app-liteFdroid-release.apk \ + app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk ``` -## Submission checklist +The [F-Droid reproducible-build documentation](https://f-droid.org/docs/Reproducible_Builds/) +describes environment differences that can affect this comparison. Run +`fdroid readmeta`, `fdroid lint`, and `fdroid build` against the current fdroiddata +checkout to validate the official build recipe as well. -1. Push the upstream metadata changes to GitHub. -2. Ensure the release commit is tagged with the normal release tag, for example `v1.7.27`. -3. Let the GitHub Release workflow upload `app-liteFdroid-release.apk` for that tag. -4. Copy the draft YAML into a fork of `fdroiddata`. -5. Run the `fdroid` validation commands locally or in the official Docker environment. -6. Open a merge request against `F-Droid/fdroiddata`. -The draft points fdroidserver at the universal F-Droid APK and keeps `Binaries` on the versioned GitHub Release asset. +## Lite APK size -## Tagging command +Lite still bundles the Japanese and English dictionaries, classified dictionaries, +English grammar data, and precomputed search indexes. Excluding AI features does +not remove those assets. -Create the release tag: +The system dictionary, English arrays, English lookup index, and grammar index +are stored uncompressed because runtime readers use `AssetManager.openFd()` and +memory mapping. English trie labels use a small alphabet and one-byte indices in +the APK; the reader restores the original UTF-16 `CharArray`, including non-ASCII +characters. Alphabets larger than 256 characters use the raw UTF-16 format. +No extracted dictionary cache is needed. -```bash -git tag v1.7.27 -git push origin v1.7.27 -``` +Release variants package the generated dictionaries instead of also shipping the +old English and Japanese system dictionary files. Connection matrix assets are +read through streams and can use APK compression. `verify-lite-apk.py` checks +these packaging rules and rejects native libraries or AI model files in Lite. + +## Store metadata -The repository's GitHub release workflow triggers on tags that start with `v`, so this tag is also where the `Binaries` URL resolves. +F-Droid imports the listing text from `fastlane/metadata/android/en-US/` and +`fastlane/metadata/android/ja-JP/`, and the listing images from the existing +`en-US/images/` subtree. diff --git a/tools/release/test_verify_lite_apk.py b/tools/release/test_verify_lite_apk.py new file mode 100644 index 000000000..9b1bb97a6 --- /dev/null +++ b/tools/release/test_verify_lite_apk.py @@ -0,0 +1,60 @@ +import contextlib +import importlib.util +import io +from pathlib import Path +import tempfile +import unittest +import zipfile + +spec = importlib.util.spec_from_file_location('lite_apk', Path(__file__).with_name('verify-lite-apk.py')) +lite_apk = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lite_apk) + + +class LiteApkTest(unittest.TestCase): + def fixture(self, directory, *, compressed=None, extra=None, bad_header=None, missing=None, stored_stream=None): + path = Path(directory) / 'app-lite.apk' + with zipfile.ZipFile(path, 'w') as archive: + for name, magic in lite_apk.MAPPED_ASSETS.items(): + if name != missing: + archive.writestr(f'assets/{name}', b'OLD1' if name == bad_header else magic, + compress_type=zipfile.ZIP_DEFLATED if name == compressed else zipfile.ZIP_STORED) + for name in lite_apk.STREAMED_ASSETS: + archive.writestr(f'assets/{name}', b'connection data', + compress_type=zipfile.ZIP_STORED if name == stored_stream else zipfile.ZIP_DEFLATED) + if extra: + archive.writestr(extra, b'unexpected') + return path + + def test_valid_packaging_passes(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + lite_apk.verify_apk(self.fixture(directory)) + + def test_compressed_mapped_asset_fails(self): + for name in lite_apk.MAPPED_ASSETS: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, 'memory-mapped asset'): + lite_apk.verify_apk(self.fixture(directory, compressed=name)) + + def test_missing_or_old_dictionary_fails(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(KeyError): + lite_apk.verify_apk(self.fixture(directory, missing='english/reading.compact')) + with self.assertRaisesRegex(ValueError, 'unexpected dictionary format'): + lite_apk.verify_apk(self.fixture(directory, bad_header='english/reading.compact')) + + def test_uncompressed_streamed_asset_fails(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, 'streamed connection asset'): + lite_apk.verify_apk(self.fixture(directory, stored_stream=lite_apk.STREAMED_ASSETS[0])) + + def test_legacy_copies_native_libraries_and_models_fail(self): + for name in [f'assets/{asset}' for asset in lite_apk.LEGACY_ASSETS] + [ + 'lib/arm64-v8a/libexample.so', 'assets/model.gguf', 'assets/model.tflite', 'assets/model.litertlm']: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): + lite_apk.verify_apk(self.fixture(directory, extra=name)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/release/verify-lite-apk.py b/tools/release/verify-lite-apk.py new file mode 100644 index 000000000..f0c64670b --- /dev/null +++ b/tools/release/verify-lite-apk.py @@ -0,0 +1,55 @@ +"""Check the runtime dictionary packaging of release Lite APKs.""" +import argparse +from pathlib import Path +import zipfile + + +# openFd() and the dictionary readers require these assets to be stored. +MAPPED_ASSETS = { + 'system/system.compact.kdict': b'KDS1', + 'english/reading.compact': b'KER2', + 'english/word.compact': b'KEW2', + 'english/token.compact': b'KET1', + 'english/english.lookup.index': b'KEI1', + 'english/grammar/categories.erol': b'EROL', +} +STREAMED_ASSETS = ('connection/connection.compact', 'connection/connection.compact.index') +LEGACY_ASSETS = ( + 'system/tango.dat.zip', 'system/yomi.dat.zip', 'system/token.dat.zip', + 'english/reading.dat.zip', 'english/word.dat', 'english/token.dat.zip', +) + + +def verify_apk(apk): + with zipfile.ZipFile(apk) as archive: + for asset, magic in MAPPED_ASSETS.items(): + name = f'assets/{asset}' + info = archive.getinfo(name) + if info.compress_type != zipfile.ZIP_STORED: + raise ValueError(f'{apk}: memory-mapped asset is compressed: {name}') + with archive.open(info) as stream: + if stream.read(4) != magic: + raise ValueError(f'{apk}: unexpected dictionary format: {name}') + for asset in STREAMED_ASSETS: + info = archive.getinfo(f'assets/{asset}') + if info.compress_type != zipfile.ZIP_DEFLATED: + raise ValueError(f'{apk}: streamed connection asset is not compressed: {asset}') + for asset in LEGACY_ASSETS: + if f'assets/{asset}' in archive.namelist(): + raise ValueError(f'{apk}: duplicate legacy dictionary: {asset}') + for name in archive.namelist(): + if name.startswith('lib/') or name.lower().endswith(('.gguf', '.tflite', '.litertlm')): + raise ValueError(f'{apk}: unexpected native library or AI model in Lite: {name}') + print(f'{apk}: runtime dictionary packaging verified; {apk.stat().st_size:,} bytes') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('apks', type=Path, nargs='+') + args = parser.parse_args() + for apk in args.apks: + verify_apk(apk) + + +if __name__ == '__main__': + main() From e8123ec4a179bcbf93faec3a2f2d3f712e8acdb7 Mon Sep 17 00:00:00 2001 From: Codex Mozc Autonomy Date: Sat, 14 Nov 2026 14:24:05 -0500 Subject: [PATCH 2/2] =?UTF-8?q?=E8=BE=9E=E6=9B=B8=E3=82=92XZ=E5=9C=A7?= =?UTF-8?q?=E7=B8=AE=E3=81=97=E3=81=A6Lite=20APK=E3=82=92=E7=B4=8458=20MB?= =?UTF-8?q?=E3=81=AB=E7=B8=AE=E5=B0=8F=20/=20Compress=20bundled=20dictiona?= =?UTF-8?q?ries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/android.yml | 1 + app/build.gradle | 32 ++++- .../assets/software-licenses/catalog.json | 33 +++++ .../assets/software-licenses/xz-1.12-0bsd.txt | 13 ++ .../compact/CompactEnglishDictionary.kt | 13 +- .../english/phrase/RuntimeGrammarRoles.kt | 2 + .../DictionaryBinaryReader.kt | 13 +- .../DictionarySourceResolver.kt | 41 ++++++ .../MappedDictionaryAssetCache.kt | 104 +++++++++++++++ .../MappedDictionaryAssetCacheTest.kt | 123 ++++++++++++++++++ docs/fdroid-release.md | 36 +++-- docs/licenses/README.md | 2 +- docs/licenses/dependency-metadata.json | 33 +++++ tools/licenses/prepare-libraries.py | 2 +- tools/release/test_verify_lite_apk.py | 23 +++- tools/release/verify-lite-apk.py | 35 ++++- 16 files changed, 471 insertions(+), 35 deletions(-) create mode 100644 app/src/main/assets/software-licenses/xz-1.12-0bsd.txt create mode 100644 app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCache.kt create mode 100644 app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCacheTest.kt diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index b9fba86ea..426c6e2d9 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -186,6 +186,7 @@ jobs: run: | ./gradlew :app:testLiteFdroidDebugUnitTest \ --tests '*BundledEnglishDictionaryReaderTest' \ + --tests '*MappedDictionaryAssetCacheTest' \ --no-daemon --max-workers=1 - name: Upload unsigned F-Droid APK for comparison diff --git a/app/build.gradle b/app/build.gradle index 608ebd8b9..f4a47ebb4 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -5,6 +5,11 @@ import org.gradle.process.ExecOperations import com.android.build.api.artifact.SingleArtifact import com.android.build.gradle.internal.tasks.ExtractNativeDebugMetadataTask +buildscript { + repositories { mavenCentral() } + dependencies { classpath 'org.tukaani:xz:1.12' } +} + plugins { id 'com.android.application' id 'com.google.dagger.hilt.android' @@ -151,10 +156,10 @@ android { } androidResources { - // Keep only memory-mapped assets stored. The connection matrix is read - // through streams, so its compact data and index can still be compressed. + // Debug assets remain directly mappable. Releases replace these six + // dictionaries with XZ streams and map their verified private copies. noCompress += ['kdict', 'reading.compact', 'word.compact', 'token.compact', - 'english.lookup.index', 'erol'] + 'english.lookup.index', 'erol', 'xz'] } sourceSets { @@ -232,6 +237,26 @@ abstract class PackageRuntimeDictionaryAssets extends DefaultTask { exclude('english/reading.dat.zip', 'english/word.dat', 'english/token.dat.zip') exclude('system/tango.dat.zip', 'system/yomi.dat.zip', 'system/token.dat.zip') } + def records = [:] + ['system/system.compact.kdict', 'english/reading.compact', 'english/word.compact', + 'english/token.compact', 'english/english.lookup.index', 'english/grammar/categories.erol'].each { path -> + def source = new File(packagedAssets.get().asFile, path) + def digest = java.security.MessageDigest.getInstance('SHA-256') + source.withInputStream { input -> + new java.security.DigestInputStream(input, digest).withCloseable { hashed -> + new File(source.path + '.xz').withOutputStream { destination -> + new org.tukaani.xz.XZOutputStream(destination, + new org.tukaani.xz.LZMA2Options(6), org.tukaani.xz.XZ.CHECK_CRC64).withCloseable { compressed -> + hashed.transferTo(compressed) + } + } + } + } + records[path] = [size: source.length(), sha256: digest.digest().encodeHex().toString()] + if (!source.delete()) throw new GradleException("Could not remove uncompressed dictionary: $path") + } + new File(packagedAssets.get().asFile, 'dictionary-assets.json').text = + groovy.json.JsonOutput.toJson([version: 1, assets: records]) + '\n' } } @@ -1341,6 +1366,7 @@ tasks.named("preBuild").configure { } dependencies { + implementation 'org.tukaani:xz:1.12' implementation 'androidx.profileinstaller:profileinstaller:1.4.0' implementation 'androidx.appcompat:appcompat:1.7.1' implementation 'com.google.android.material:material:1.13.0' diff --git a/app/src/main/assets/software-licenses/catalog.json b/app/src/main/assets/software-licenses/catalog.json index cf94ef3b8..c6a4be837 100644 --- a/app/src/main/assets/software-licenses/catalog.json +++ b/app/src/main/assets/software-licenses/catalog.json @@ -3686,6 +3686,39 @@ "liteFdroidRelease" ], "notices": [] + }, + { + "group": "org.tukaani", + "name": "xz", + "version": "1.12", + "pomUrl": "https://repo.maven.apache.org/maven2/org/tukaani/xz/1.12/xz-1.12.pom", + "licenses": [ + { + "name": "0BSD" + } + ], + "evidence": [ + { + "url": "https://repo.maven.apache.org/maven2/org/tukaani/xz/1.12/xz-1.12.pom", + "sha256": "937b44d869c059c39f07ae194d8cd6219f5aebf2200ef9707f0b50b7837da281" + }, + { + "url": "https://raw.githubusercontent.com/tukaani-project/xz-java/v1.12/COPYING", + "sha256": "1e77ef26cee5216d6531e49cbf2f6111a0270dc7b322fc65477063c5aff88766" + } + ], + "variants": [ + "fullStandardRelease", + "liteStandardRelease", + "liteFdroidRelease" + ], + "notices": [ + { + "title": "XZ for Java 1.12 COPYING (0BSD)", + "asset": "software-licenses/xz-1.12-0bsd.txt", + "sha256": "1e77ef26cee5216d6531e49cbf2f6111a0270dc7b322fc65477063c5aff88766" + } + ] } ] } diff --git a/app/src/main/assets/software-licenses/xz-1.12-0bsd.txt b/app/src/main/assets/software-licenses/xz-1.12-0bsd.txt new file mode 100644 index 000000000..16270b71e --- /dev/null +++ b/app/src/main/assets/software-licenses/xz-1.12-0bsd.txt @@ -0,0 +1,13 @@ +Copyright (C) The XZ for Java authors and contributors + +Permission to use, copy, modify, and/or distribute this +software for any purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL +THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt index 28e48651d..9106e7171 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/CompactEnglishDictionary.kt @@ -36,8 +36,15 @@ internal object BundledEnglishDictionaryReader { reading: LOUDSWithTermId, word: LOUDS, token: TokenArray, + ): CompactEnglishDictionary = readIndex(map(descriptor), reading, word, token) + + internal fun readIndex( + source: ByteBuffer, + reading: LOUDSWithTermId, + word: LOUDS, + token: TokenArray, ): CompactEnglishDictionary { - val input = Cursor(descriptor) + val input = Cursor(source) require(input.readInt() == MAGIC) { "Invalid English lookup index magic" } // The build indexes the used words of each BitSet. A serialized BitSet may // also retain spare words that would make its runtime rank table larger. @@ -83,7 +90,9 @@ internal object BundledEnglishDictionaryReader { LOUDS(lbs, readLabels(), leaves) } - fun readToken(descriptor: AssetFileDescriptor): TokenArray = read(map(descriptor), 0x4B455431) { + fun readToken(descriptor: AssetFileDescriptor): TokenArray = readToken(map(descriptor)) + + internal fun readToken(source: ByteBuffer): TokenArray = read(source, 0x4B455431) { TokenArray.fromArrays(readShortArray(), readIntArray(), readBitSet()) } diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/english/phrase/RuntimeGrammarRoles.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/english/phrase/RuntimeGrammarRoles.kt index 4eed189e1..b6f46b3fb 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/english/phrase/RuntimeGrammarRoles.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/english/phrase/RuntimeGrammarRoles.kt @@ -245,6 +245,8 @@ internal class RuntimeGrammarRoles private constructor( internal fun read(source: ByteBuffer): RuntimeGrammarRoles = read(source, validateAllRecords = true) + internal fun readBundled(source: ByteBuffer): RuntimeGrammarRoles = read(source, validateAllRecords = false) + private fun read(source: ByteBuffer, validateAllRecords: Boolean): RuntimeGrammarRoles { val data = source.slice().asReadOnlyBuffer().order(ByteOrder.BIG_ENDIAN) require(data.capacity() in HEADER_SIZE..MAX_INDEX_SIZE && data.getInt(0) == MAGIC && diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryBinaryReader.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryBinaryReader.kt index c51d46a29..2063fe60a 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryBinaryReader.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionaryBinaryReader.kt @@ -230,13 +230,13 @@ class DictionaryBinaryReader @Inject constructor( ) private fun readBundledEnglishReading(): EnglishReadingTrie = - resolver.openBundledAssetFileDescriptor("english/reading.compact").use(BundledEnglishDictionaryReader::readReading) + resolver.readBundledAssetBuffer("english/reading.compact", BundledEnglishDictionaryReader::readReading) private fun readBundledEnglishWord(): EnglishWordTrie = - resolver.openBundledAssetFileDescriptor("english/word.compact").use(BundledEnglishDictionaryReader::readWord) + resolver.readBundledAssetBuffer("english/word.compact", BundledEnglishDictionaryReader::readWord) private fun readBundledEnglishToken(): EnglishTokenArray = - resolver.openBundledAssetFileDescriptor("english/token.compact").use(BundledEnglishDictionaryReader::readToken) + resolver.readBundledAssetBuffer("english/token.compact", BundledEnglishDictionaryReader::readToken) private fun readEnglishReading(input: InputStream): EnglishReadingTrie = openZipAwareObjectInputStream(input, DictionaryFileKey.ENGLISH_READING.name).use { @@ -346,7 +346,7 @@ class DictionaryBinaryReader @Inject constructor( if (compactSystemDictionaryLoadAttempted) return@synchronized null compactSystemDictionaryLoadAttempted = true runCatching { - resolver.openBundledAssetFileDescriptor(COMPACT_SYSTEM_DICTIONARY_ASSET).use( + resolver.readBundledAssetBuffer(COMPACT_SYSTEM_DICTIONARY_ASSET, CompactSystemDictionaryReader::read, ) }.onFailure { error -> @@ -363,8 +363,7 @@ class DictionaryBinaryReader @Inject constructor( internal fun loadBundledEnglishGrammar(): GrammarRoles? = runCatching { - resolver.openBundledAssetFileDescriptor("english/grammar/categories.erol") - .use(RuntimeGrammarRoles::read) + resolver.readBundledAssetBuffer("english/grammar/categories.erol", RuntimeGrammarRoles::readBundled) }.onFailure { error -> Timber.w(error, "Bundled English grammar is unavailable") }.getOrNull() @@ -377,7 +376,7 @@ class DictionaryBinaryReader @Inject constructor( compactEnglishDictionaryCache?.let { return@synchronized it } if (compactEnglishDictionaryLoadAttempted) return@synchronized null val loaded = runCatching { - resolver.openBundledAssetFileDescriptor(ENGLISH_LOOKUP_INDEX_ASSET).use { index -> + resolver.readBundledAssetBuffer(ENGLISH_LOOKUP_INDEX_ASSET) { index -> val reading = readBundledEnglishReading() val word = readBundledEnglishWord() val token = readBundledEnglishToken() diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt index 7dbbec071..5b08c8a43 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/DictionarySourceResolver.kt @@ -4,6 +4,13 @@ import android.content.Context import android.content.res.AssetFileDescriptor import dagger.hilt.android.qualifiers.ApplicationContext import java.io.InputStream +import java.io.File +import java.io.FileInputStream +import java.io.FileNotFoundException +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.channels.FileChannel +import timber.log.Timber import javax.inject.Inject import javax.inject.Singleton @@ -12,6 +19,40 @@ class DictionarySourceResolver @Inject constructor( @ApplicationContext private val context: Context, private val store: DictionaryOverrideStore, ) { + private val mappedDictionaryCache: MappedDictionaryAssetCache? by lazy { + val manifest = try { + context.assets.open("dictionary-assets.json").bufferedReader().use { it.readText() } + } catch (_: FileNotFoundException) { + null // Debug builds retain the uncompressed, directly mappable assets. + } + manifest?.let { + MappedDictionaryAssetCache(File(context.noBackupFilesDir, "bundled-dictionaries"), it) { path -> + context.assets.open(path) + } + } + } + + internal fun readBundledAssetBuffer(path: String, read: (ByteBuffer) -> T): T { + val cache = mappedDictionaryCache + if (cache != null && cache.contains(path)) { + val file = try { + cache.fileFor(path) + } catch (error: IOException) { + Timber.w(error, "Cannot cache bundled dictionary; decoding in memory: %s", path) + null + } + if (file == null) return read(ByteBuffer.wrap(cache.readBytes(path))) + return FileInputStream(file).channel.use { channel -> + read(channel.map(FileChannel.MapMode.READ_ONLY, 0, file.length())) + } + } + return openBundledAssetFileDescriptor(path).use { descriptor -> + FileInputStream(descriptor.fileDescriptor).channel.use { channel -> + read(channel.map(FileChannel.MapMode.READ_ONLY, descriptor.startOffset, descriptor.length)) + } + } + } + fun openForKey(key: DictionaryFileKey): InputStream = if (shouldUseOverride(key)) openOverrideForKey(key) else openBundledForKey(key) diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCache.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCache.kt new file mode 100644 index 000000000..6299e1196 --- /dev/null +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCache.kt @@ -0,0 +1,104 @@ +package com.kazumaproject.markdownhelperkeyboard.dictionary_override + +import com.google.gson.JsonParser +import java.io.File +import java.io.FileOutputStream +import java.io.IOException +import java.io.InputStream +import java.io.RandomAccessFile +import java.security.MessageDigest +import org.tukaani.xz.XZInputStream + +/** Private, reproducible dictionary copies used by the existing memory-mapped readers. */ +internal class MappedDictionaryAssetCache( + private val directory: File, + manifest: String, + private val openAsset: (String) -> InputStream, +) { + private data class Asset(val size: Long, val sha256: String) + + private val assets: Map = JsonParser.parseString(manifest).asJsonObject.let { root -> + require(root.get("version").asInt == 1) { "Unsupported dictionary asset manifest" } + root.getAsJsonObject("assets").entrySet().associate { (path, value) -> + val record = value.asJsonObject + val asset = Asset(record.get("size").asLong, record.get("sha256").asString) + require(asset.size in 1..MAX_ASSET_BYTES && asset.sha256.matches(Regex("[0-9a-f]{64}"))) { + "Invalid dictionary asset record: $path" + } + path to asset + }.also { require(it.isNotEmpty()) { "Empty dictionary asset manifest" } } + } + + fun contains(path: String): Boolean = path in assets + + fun fileFor(path: String): File = synchronized(processLock) { + val asset = assets.getValue(path) + if (!directory.isDirectory && !directory.mkdirs()) { + throw IOException("Cannot create private dictionary directory") + } + // A process lock also prevents overlapping JVM file locks in callers on other threads. + RandomAccessFile(File(directory, ".lock"), "rw").channel.use { channel -> + channel.lock().use { + val liveFiles = assets.values.mapTo(HashSet()) { it.sha256 } + directory.listFiles()?.filter { + it.name != ".lock" && it.name !in liveFiles + }?.forEach { it.delete() } + val destination = File(directory, asset.sha256) + if (destination.isFile && destination.length() == asset.size) return destination + val temporary = File.createTempFile("dictionary-", ".tmp", directory) + try { + FileOutputStream(temporary).use { output -> + copyVerified(path) { bytes, count -> output.write(bytes, 0, count) } + output.fd.sync() + } + if (!temporary.setReadOnly() || !temporary.renameTo(destination)) { + throw IOException("Cannot publish private dictionary: $path") + } + return destination + } finally { + temporary.delete() + } + } + } + } + + /** A full/read-only filesystem must not make the bundled dictionary unavailable. */ + fun readBytes(path: String): ByteArray { + val bytes = ByteArray(assets.getValue(path).size.toInt()) + var offset = 0 + copyVerified(path) { buffer, count -> + buffer.copyInto(bytes, offset, 0, count) + offset += count + } + return bytes + } + + private fun copyVerified(path: String, consume: (ByteArray, Int) -> Unit) { + val asset = assets.getValue(path) + val digest = MessageDigest.getInstance("SHA-256") + var size = 0L + openAsset("$path.xz").use { raw -> + // The build uses an 8 MiB dictionary; bound decoder memory for damaged streams too. + XZInputStream(raw, 32 * 1024).use { input -> + val buffer = ByteArray(64 * 1024) + while (true) { + val count = input.read(buffer) + if (count < 0) break + size += count + if (size > asset.size) throw IOException("Dictionary exceeds expected size: $path") + digest.update(buffer, 0, count) + consume(buffer, count) + } + } + } + val hash = digest.digest().joinToString("") { "%02x".format(it) } + if (size != asset.size || hash != asset.sha256) { + throw IOException("Dictionary checksum/size mismatch: $path") + } + } + + companion object { + private val processLock = Any() + private const val MAX_ASSET_BYTES = 128L * 1024 * 1024 + } +} diff --git a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCacheTest.kt b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCacheTest.kt new file mode 100644 index 000000000..d5965015e --- /dev/null +++ b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/MappedDictionaryAssetCacheTest.kt @@ -0,0 +1,123 @@ +package com.kazumaproject.markdownhelperkeyboard.dictionary_override + +import com.google.gson.JsonObject +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.IOException +import java.security.MessageDigest +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import org.junit.Assert.* +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.tukaani.xz.LZMA2Options +import org.tukaani.xz.XZOutputStream + +class MappedDictionaryAssetCacheTest { + @get:Rule val temp = TemporaryFolder() + private val path = "english/reading.compact" + private val bytes = "Dictionary \u0000 \uffff 日本語".repeat(2048).toByteArray() + + @Test fun extractsVerifiedBytesOnceAndReusesReadOnlyFile() { + val opens = AtomicInteger() + val cache = cache(temp.newFolder(), bytes, opens) + val file = cache.fileFor(path) + assertArrayEquals(bytes, file.readBytes()) + assertEquals(hash(bytes), file.name) + assertEquals(file, cache.fileFor(path)) + assertEquals(1, opens.get()) + } + + @Test fun changedDictionaryReplacesOldCopyWithoutKeepingOldVersions() { + val directory = temp.newFolder() + val old = cache(directory, bytes).fileFor(path) + val updated = bytes + byteArrayOf(1) + val file = cache(directory, updated).fileFor(path) + assertFalse(old.exists()) + assertArrayEquals(updated, file.readBytes()) + assertEquals(2, directory.listFiles()!!.size) // Current copy and process lock. + } + + @Test fun truncatedExistingCopyIsRebuilt() { + val cache = cache(temp.newFolder(), bytes) + val file = cache.fileFor(path) + assertTrue(file.setWritable(true)) + file.writeBytes(byteArrayOf(0)) + assertArrayEquals(bytes, cache.fileFor(path).readBytes()) + } + + @Test fun damagedStreamLeavesNoPartialDictionaryAndCanBeRetried() { + val directory = temp.newFolder() + val compressed = compress(bytes) + var damaged = true + val cache = MappedDictionaryAssetCache(directory, manifest(bytes)) { + ByteArrayInputStream(if (damaged) compressed.copyOf(compressed.size - 8) else compressed) + } + assertThrows(IOException::class.java) { cache.fileFor(path) } + assertEquals(listOf(".lock"), directory.listFiles()!!.map { it.name }) + damaged = false + assertArrayEquals(bytes, cache.fileFor(path).readBytes()) + } + + @Test fun wrongChecksumAndOversizedOutputAreRejected() { + for (metadata in listOf(manifest(bytes, digest = "0".repeat(64)), manifest(bytes, size = bytes.size - 1))) { + val directory = temp.newFolder() + val cache = MappedDictionaryAssetCache(directory, metadata) { ByteArrayInputStream(compress(bytes)) } + assertThrows(IOException::class.java) { cache.fileFor(path) } + assertThrows(IOException::class.java) { cache.readBytes(path) } + assertFalse(File(directory, hash(bytes)).exists()) + } + } + + @Test fun memoryFallbackWorksWhenCacheDirectoryCannotBeWritten() { + val cache = cache(temp.newFile(), bytes) + assertThrows(IOException::class.java) { cache.fileFor(path) } + assertArrayEquals(bytes, cache.readBytes(path)) + } + + @Test fun concurrentReadersAndCacheInstancesPublishOnlyOneCopy() { + val directory = temp.newFolder() + val opens = AtomicInteger() + val caches = List(4) { cache(directory, bytes, opens) } + val executor = Executors.newFixedThreadPool(4) + try { + val results = caches.map { cache -> executor.submit { cache.fileFor(path) } } + .map { it.get(30, TimeUnit.SECONDS) } + assertEquals(1, results.toSet().size) + assertEquals(1, opens.get()) + assertArrayEquals(bytes, results.first().readBytes()) + } finally { + executor.shutdownNow() + } + } + + private fun cache(directory: File, data: ByteArray, opens: AtomicInteger = AtomicInteger()): MappedDictionaryAssetCache { + val compressed = compress(data) + return MappedDictionaryAssetCache(directory, manifest(data)) { name -> + assertEquals("$path.xz", name) + opens.incrementAndGet() + ByteArrayInputStream(compressed) + } + } + + private fun manifest(data: ByteArray, digest: String = hash(data), size: Int = data.size): String = + JsonObject().apply { + addProperty("version", 1) + add("assets", JsonObject().apply { + add(path, JsonObject().apply { + addProperty("size", size) + addProperty("sha256", digest) + }) + }) + }.toString() + + private fun hash(data: ByteArray) = MessageDigest.getInstance("SHA-256").digest(data) + .joinToString("") { "%02x".format(it) } + + private fun compress(data: ByteArray): ByteArray = ByteArrayOutputStream().also { bytes -> + XZOutputStream(bytes, LZMA2Options(1)).use { it.write(data) } + }.toByteArray() +} diff --git a/docs/fdroid-release.md b/docs/fdroid-release.md index f2f9018b6..cc5d3918b 100644 --- a/docs/fdroid-release.md +++ b/docs/fdroid-release.md @@ -77,17 +77,31 @@ Lite still bundles the Japanese and English dictionaries, classified dictionarie English grammar data, and precomputed search indexes. Excluding AI features does not remove those assets. -The system dictionary, English arrays, English lookup index, and grammar index -are stored uncompressed because runtime readers use `AssetManager.openFd()` and -memory mapping. English trie labels use a small alphabet and one-byte indices in -the APK; the reader restores the original UTF-16 `CharArray`, including non-ASCII -characters. Alphabets larger than 256 characters use the raw UTF-16 format. -No extracted dictionary cache is needed. - -Release variants package the generated dictionaries instead of also shipping the -old English and Japanese system dictionary files. Connection matrix assets are -read through streams and can use APK compression. `verify-lite-apk.py` checks -these packaging rules and rejects native libraries or AI model files in Lite. +Release APKs store the six memory-mapped dictionaries and indexes as XZ streams. +The encoder and decoder both use the pinned pure-Java `org.tukaani:xz:1.12`; +the build emits an asset manifest with the uncompressed sizes and SHA-256 hashes. +It includes no timestamps or machine paths. + +On first use of an asset, the app verifies and atomically writes a read-only copy +to its private `noBackupFilesDir/bundled-dictionaries/` directory, then uses the +existing memory-mapped reader. Later processes reuse copies whose content-addressed +name and size match the packaged manifest. Changed assets replace obsolete copies +under a cross-process file lock. The cache is not included in backups. + +This reduces APK/download size. Expanding all six assets requires about 65.73 MB +of additional private storage and adds work on first use after a dictionary change. +If a cache write fails, the reader decompresses and verifies the asset in memory; +that fallback uses more heap until its source buffer is no longer needed (lookup +bounds and grammar retain their backing buffers). + +Debug variants retain directly mappable assets. English trie labels use a small +alphabet and one-byte indices; the reader restores the original UTF-16 arrays. +Alphabets larger than 256 characters use raw UTF-16. + +Release variants exclude duplicate legacy Japanese system and English dictionary +files. Streamed connection matrices use APK compression. `verify-lite-apk.py` +decodes all six XZ streams, checks their sizes, hashes and format headers, and +rejects duplicate or unexpected native/model assets. ## Store metadata diff --git a/docs/licenses/README.md b/docs/licenses/README.md index 17bb73813..8c391a201 100644 --- a/docs/licenses/README.md +++ b/docs/licenses/README.md @@ -35,7 +35,7 @@ MITは第三者の辞書・ライブラリ・モデルのライセンスを変 アプリの「ライセンスと出典」は、概要、辞書・データ、ライブラリ、Full版のAIモデル・ネイティブコードに分けて表示します。辞書の詳細では同梱版の該当する通知原文をオフラインで閲覧できます。追加語と対象層分類の出典は各カテゴリの詳細に、英単語辞書の生成元と文法データの入力原本は英単語辞書の詳細に掲載しています。 -ライブラリはFull Standard Releaseの138件、Lite Standard/F-Droid Releaseの136件の解決結果に対応します(BOM等のメタデータ依存を含む)。版別一覧とPOMの根拠は `app/src/main/assets/software-licenses/catalog.json` に保持しています。対象版のPOM照合に加え、取得できたAAR/JARとそのclasses.jarのLICENSE/NOTICE/COPYINGを原文のまま保存し、表示します。解決されたモジュールと縮小後APKの到達可能コードとの一対一対応を示すものではありません。 +ライブラリはFull Standard Releaseの139件、Lite Standard/F-Droid Releaseの137件の解決結果に対応します(BOM等のメタデータ依存を含む)。版別一覧とPOMの根拠は `app/src/main/assets/software-licenses/catalog.json` に保持しています。対象版のPOM照合に加え、取得できたAAR/JARとそのclasses.jarのLICENSE/NOTICE/COPYINGを原文のまま保存し、表示します。解決されたモジュールと縮小後APKの到達可能コードとの一対一対応を示すものではありません。 コルーチンは対象版のPOMに基づくApache 2.0、Zenzモデルは固定版モデルカードのApache 2.0として表示します。根拠のない一律の著作権表示は付与しません。個別著作権表示を取得できない場合は、その範囲を詳細画面に明記します。ユーザーが追加するモデルの利用条件は提供元の資料によります。 diff --git a/docs/licenses/dependency-metadata.json b/docs/licenses/dependency-metadata.json index 7ee337b7c..9c0db2734 100644 --- a/docs/licenses/dependency-metadata.json +++ b/docs/licenses/dependency-metadata.json @@ -3118,6 +3118,39 @@ "sha256": "63c37501388202aea61fe5f471542c61634ba90a2c101c40290b65339e6cc8e8" } ] + }, + { + "group": "org.tukaani", + "name": "xz", + "version": "1.12", + "configurations": [ + "fullStandardReleaseRuntimeClasspath", + "liteStandardReleaseRuntimeClasspath", + "liteFdroidReleaseRuntimeClasspath" + ], + "pomUrl": "https://repo.maven.apache.org/maven2/org/tukaani/xz/1.12/xz-1.12.pom", + "licenses": [ + { + "name": "0BSD" + } + ], + "evidence": [ + { + "url": "https://repo.maven.apache.org/maven2/org/tukaani/xz/1.12/xz-1.12.pom", + "sha256": "937b44d869c059c39f07ae194d8cd6219f5aebf2200ef9707f0b50b7837da281" + }, + { + "url": "https://raw.githubusercontent.com/tukaani-project/xz-java/v1.12/COPYING", + "sha256": "1e77ef26cee5216d6531e49cbf2f6111a0270dc7b322fc65477063c5aff88766" + } + ], + "sourceNotices": [ + { + "title": "XZ for Java 1.12 COPYING (0BSD)", + "asset": "software-licenses/xz-1.12-0bsd.txt", + "sha256": "1e77ef26cee5216d6531e49cbf2f6111a0270dc7b322fc65477063c5aff88766" + } + ] } ] } diff --git a/tools/licenses/prepare-libraries.py b/tools/licenses/prepare-libraries.py index ed7863ffb..84644923c 100644 --- a/tools/licenses/prepare-libraries.py +++ b/tools/licenses/prepare-libraries.py @@ -73,7 +73,7 @@ def notices(data, prefix, result): source_notices = original.get('sourceNotices', []) names = {license['name'].lower() for license in original['licenses']} assert (names and all('apache' in name for name in names)) or ( - names == {'mit license'} and source_notices + names in ({'mit license'}, {'0bsd'}) and source_notices ), f'Audit license text mapping before adding {key}' for notice in source_notices: verify_notice(notice) diff --git a/tools/release/test_verify_lite_apk.py b/tools/release/test_verify_lite_apk.py index 9b1bb97a6..208d3fa57 100644 --- a/tools/release/test_verify_lite_apk.py +++ b/tools/release/test_verify_lite_apk.py @@ -1,6 +1,9 @@ import contextlib import importlib.util import io +import json +import hashlib +import lzma from pathlib import Path import tempfile import unittest @@ -12,13 +15,17 @@ class LiteApkTest(unittest.TestCase): - def fixture(self, directory, *, compressed=None, extra=None, bad_header=None, missing=None, stored_stream=None): + def fixture(self, directory, *, compressed=None, extra=None, bad_header=None, missing=None, stored_stream=None, bad_digest=False): path = Path(directory) / 'app-lite.apk' + manifest = {'version': 1, 'assets': {}} with zipfile.ZipFile(path, 'w') as archive: for name, magic in lite_apk.MAPPED_ASSETS.items(): + data = b'OLD1' if name == bad_header else magic + manifest['assets'][name] = {'size': len(data), 'sha256': '0' * 64 if bad_digest else hashlib.sha256(data).hexdigest()} if name != missing: - archive.writestr(f'assets/{name}', b'OLD1' if name == bad_header else magic, + archive.writestr(f'assets/{name}.xz', lzma.compress(data), compress_type=zipfile.ZIP_DEFLATED if name == compressed else zipfile.ZIP_STORED) + archive.writestr('assets/dictionary-assets.json', json.dumps(manifest)) for name in lite_apk.STREAMED_ASSETS: archive.writestr(f'assets/{name}', b'connection data', compress_type=zipfile.ZIP_STORED if name == stored_stream else zipfile.ZIP_DEFLATED) @@ -33,7 +40,7 @@ def test_valid_packaging_passes(self): def test_compressed_mapped_asset_fails(self): for name in lite_apk.MAPPED_ASSETS: with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: - with self.assertRaisesRegex(ValueError, 'memory-mapped asset'): + with self.assertRaisesRegex(ValueError, 'compressed again'): lite_apk.verify_apk(self.fixture(directory, compressed=name)) def test_missing_or_old_dictionary_fails(self): @@ -48,6 +55,16 @@ def test_uncompressed_streamed_asset_fails(self): with self.assertRaisesRegex(ValueError, 'streamed connection asset'): lite_apk.verify_apk(self.fixture(directory, stored_stream=lite_apk.STREAMED_ASSETS[0])) + def test_wrong_digest_fails(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, 'checksum/size mismatch'): + lite_apk.verify_apk(self.fixture(directory, bad_digest=True)) + + def test_duplicate_uncompressed_dictionary_fails(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaisesRegex(ValueError, 'duplicate uncompressed dictionary'): + lite_apk.verify_apk(self.fixture(directory, extra='assets/english/reading.compact')) + def test_legacy_copies_native_libraries_and_models_fail(self): for name in [f'assets/{asset}' for asset in lite_apk.LEGACY_ASSETS] + [ 'lib/arm64-v8a/libexample.so', 'assets/model.gguf', 'assets/model.tflite', 'assets/model.litertlm']: diff --git a/tools/release/verify-lite-apk.py b/tools/release/verify-lite-apk.py index f0c64670b..108c6028a 100644 --- a/tools/release/verify-lite-apk.py +++ b/tools/release/verify-lite-apk.py @@ -1,10 +1,12 @@ -"""Check the runtime dictionary packaging of release Lite APKs.""" +"""Check compressed runtime dictionaries in release Lite APKs.""" import argparse +import hashlib +import json +import lzma from pathlib import Path import zipfile -# openFd() and the dictionary readers require these assets to be stored. MAPPED_ASSETS = { 'system/system.compact.kdict': b'KDS1', 'english/reading.compact': b'KER2', @@ -22,14 +24,33 @@ def verify_apk(apk): with zipfile.ZipFile(apk) as archive: + manifest = json.loads(archive.read('assets/dictionary-assets.json')) + if manifest['version'] != 1 or set(manifest['assets']) != set(MAPPED_ASSETS): + raise ValueError(f'{apk}: invalid dictionary asset manifest') for asset, magic in MAPPED_ASSETS.items(): - name = f'assets/{asset}' + if f'assets/{asset}' in archive.namelist(): + raise ValueError(f'{apk}: duplicate uncompressed dictionary: {asset}') + name = f'assets/{asset}.xz' info = archive.getinfo(name) if info.compress_type != zipfile.ZIP_STORED: - raise ValueError(f'{apk}: memory-mapped asset is compressed: {name}') - with archive.open(info) as stream: - if stream.read(4) != magic: + raise ValueError(f'{apk}: XZ dictionary is compressed again by ZIP: {name}') + record = manifest['assets'][asset] + if not 0 < record['size'] <= 128 * 1024 * 1024: + raise ValueError(f'{apk}: invalid dictionary size: {asset}') + size = 0 + digest = hashlib.sha256() + with archive.open(info) as raw, lzma.LZMAFile(raw) as stream: + if (header := stream.read(4)) != magic: raise ValueError(f'{apk}: unexpected dictionary format: {name}') + size += len(header) + digest.update(header) + while data := stream.read(64 * 1024): + size += len(data) + if size > record['size']: + raise ValueError(f'{apk}: dictionary exceeds expected size: {name}') + digest.update(data) + if size != record['size'] or digest.hexdigest() != record['sha256']: + raise ValueError(f'{apk}: dictionary checksum/size mismatch: {name}') for asset in STREAMED_ASSETS: info = archive.getinfo(f'assets/{asset}') if info.compress_type != zipfile.ZIP_DEFLATED: @@ -40,7 +61,7 @@ def verify_apk(apk): for name in archive.namelist(): if name.startswith('lib/') or name.lower().endswith(('.gguf', '.tflite', '.litertlm')): raise ValueError(f'{apk}: unexpected native library or AI model in Lite: {name}') - print(f'{apk}: runtime dictionary packaging verified; {apk.stat().st_size:,} bytes') + print(f'{apk}: compressed runtime dictionary packaging verified; {apk.stat().st_size:,} bytes') def main():