Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions .github/workflows/android.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,20 @@ name: Android Release CI

on:
workflow_dispatch:
pull_request:
paths:
- '.github/workflows/android.yml'
- 'build.gradle'
- 'settings.gradle'
- 'gradle.properties'
- 'gradle/**'
- 'app/build.gradle'
- 'app/src/main/assets/**'
- 'app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/**'
- 'app/src/main/java/com/kazumaproject/markdownhelperkeyboard/dictionary_override/**'
- 'app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/compact/**'
- 'tools/release/**'
- 'tools/classified-dictionaries/**'
push:
tags:
- 'v*'
Expand All @@ -15,6 +29,7 @@ concurrency:

jobs:
build:
if: github.event_name != 'pull_request'
runs-on: ubuntu-latest
env:
ZENZ_MODEL_CACHE_DIR: ${{ github.workspace }}/.zenz-model-cache
Expand Down Expand Up @@ -92,6 +107,7 @@ jobs:
--stacktrace \
--no-daemon \
--max-workers=1
python3 tools/release/verify-lite-apk.py app/build/outputs/apk/liteStandard/release/*.apk

- name: Build and verify Play bundle native symbols
env:
Expand Down Expand Up @@ -147,7 +163,7 @@ jobs:
run: sdkmanager "platforms;android-37.0" "build-tools;36.0.0" "ndk;29.0.14206865"

- name: Verify release source guard
run: python3 -m unittest discover -s tools/release -p 'test_verify_release_source.py'
run: python3 -m unittest discover -s tools/release

- name: Build unsigned F-Droid APK without restored caches
env:
Expand All @@ -161,6 +177,17 @@ jobs:
--stacktrace --no-daemon --max-workers=1
python3 tools/release/verify-release-source.py --revision "$GITHUB_SHA" \
app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk
python3 tools/release/verify-lite-apk.py \
app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk

- name: Test English dictionary decoding
env:
GRADLE_USER_HOME: ${{ runner.temp }}/fdroid-gradle-${{ matrix.build }}
run: |
./gradlew :app:testLiteFdroidDebugUnitTest \
--tests '*BundledEnglishDictionaryReaderTest' \
--tests '*MappedDictionaryAssetCacheTest' \
--no-daemon --max-workers=1

- name: Upload unsigned F-Droid APK for comparison
uses: actions/upload-artifact@v4
Expand All @@ -169,8 +196,24 @@ jobs:
if-no-files-found: error
path: app/build/outputs/apk/liteFdroid/release/app-liteFdroid-release.apk

verify-fdroid:
needs: fdroid
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
name: fdroid-unsigned-first-${{ github.sha }}
path: fdroid/first
- uses: actions/download-artifact@v4
with:
name: fdroid-unsigned-second-${{ github.sha }}
path: fdroid/second
- name: Require identical clean F-Droid builds
run: cmp fdroid/first/app-liteFdroid-release.apk fdroid/second/app-liteFdroid-release.apk

publish:
needs: [build, fdroid]
if: github.event_name != 'pull_request'
needs: [build, fdroid, verify-fdroid]
runs-on: ubuntu-latest
steps:
- name: Checkout verification tools from the workflow commit
Expand Down
72 changes: 57 additions & 15 deletions app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ import org.gradle.process.ExecOperations
import com.android.build.api.artifact.SingleArtifact
import com.android.build.gradle.internal.tasks.ExtractNativeDebugMetadataTask

buildscript {
repositories { mavenCentral() }
dependencies { classpath 'org.tukaani:xz:1.12' }
}

plugins {
id 'com.android.application'
id 'com.google.dagger.hilt.android'
Expand Down Expand Up @@ -151,9 +156,10 @@ android {
}

androidResources {
// These assets are already compact binary images. Keeping them stored lets
// Android read them directly instead of inflating them on the IME's cold path.
noCompress += ['kdict', 'compact', 'index', 'erol']
// Debug assets remain directly mappable. Releases replace these six
// dictionaries with XZ streams and map their verified private copies.
noCompress += ['kdict', 'reading.compact', 'word.compact', 'token.compact',
'english.lookup.index', 'erol', 'xz']
}

sourceSets {
Expand Down Expand Up @@ -213,9 +219,6 @@ abstract class GenerateDictionaryLicenseAssets extends DefaultTask {
}

abstract class PackageRuntimeDictionaryAssets extends DefaultTask {
@Input
abstract Property<Boolean> getStandardChannel()

@InputDirectory
@PathSensitive(PathSensitivity.RELATIVE)
abstract DirectoryProperty getMergedAssets()
Expand All @@ -232,10 +235,28 @@ abstract class PackageRuntimeDictionaryAssets extends DefaultTask {
from(mergedAssets)
into(packagedAssets)
exclude('english/reading.dat.zip', 'english/word.dat', 'english/token.dat.zip')
if (standardChannel.get()) {
exclude('system/tango.dat.zip', 'system/yomi.dat.zip', 'system/token.dat.zip')
exclude('system/tango.dat.zip', 'system/yomi.dat.zip', 'system/token.dat.zip')
}
def records = [:]
['system/system.compact.kdict', 'english/reading.compact', 'english/word.compact',
'english/token.compact', 'english/english.lookup.index', 'english/grammar/categories.erol'].each { path ->
def source = new File(packagedAssets.get().asFile, path)
def digest = java.security.MessageDigest.getInstance('SHA-256')
source.withInputStream { input ->
new java.security.DigestInputStream(input, digest).withCloseable { hashed ->
new File(source.path + '.xz').withOutputStream { destination ->
new org.tukaani.xz.XZOutputStream(destination,
new org.tukaani.xz.LZMA2Options(6), org.tukaani.xz.XZ.CHECK_CRC64).withCloseable { compressed ->
hashed.transferTo(compressed)
}
}
}
}
records[path] = [size: source.length(), sha256: digest.digest().encodeHex().toString()]
if (!source.delete()) throw new GradleException("Could not remove uncompressed dictionary: $path")
}
new File(packagedAssets.get().asFile, 'dictionary-assets.json').text =
groovy.json.JsonOutput.toJson([version: 1, assets: records]) + '\n'
}
}

Expand Down Expand Up @@ -279,9 +300,7 @@ androidComponents {
}
variant.sources.assets.addGeneratedSourceDirectory(licenses) { it.outputDirectory }
def compactAssets = tasks.register("package${variant.name.capitalize()}RuntimeDictionaryAssets",
PackageRuntimeDictionaryAssets) {
standardChannel.set(variant.productFlavors.any { it.first == 'channel' && it.second == 'standard' })
}
PackageRuntimeDictionaryAssets)
variant.artifacts.use(compactAssets)
.wiredWithDirectories({ it.mergedAssets }, { it.packagedAssets })
.toTransform(SingleArtifact.ASSETS.INSTANCE)
Expand Down Expand Up @@ -757,19 +776,41 @@ tasks.register("generateCompactSystemDictionary") {
write(output)
}
}
writeEntries("reading.compact", 0x4B455231) { output -> // KER1
// English tries use a small alphabet. Store one-byte alphabet indices,
// then restore the same CharArray at load time. Larger alphabets keep UTF-16.
def writeLabels = { DataOutputStream output, char[] labels ->
char[] alphabet = new String(labels).chars().distinct().sorted().toArray()
.collect { (char) it } as char[]
if (alphabet.length == 0 || alphabet.length > 256) {
output.writeInt(0)
writeArray(output, labels.length, Character.BYTES) { it.asCharBuffer().put(labels) }
} else {
writeArray(output, alphabet.length, Character.BYTES) { it.asCharBuffer().put(alphabet) }
byte[] indexByChar = new byte[Character.MAX_VALUE + 1]
for (int index = 0; index < alphabet.length; index++) {
indexByChar[(int) alphabet[index]] = (byte) index
}
byte[] indices = new byte[labels.length]
for (int index = 0; index < labels.length; index++) {
indices[index] = indexByChar[(int) labels[index]]
}
output.writeInt(indices.length)
output.write(indices)
}
}
writeEntries("reading.compact", 0x4B455232) { output -> // KER2
writeBitSet(output, englishReading[0] as BitSet)
writeBitSet(output, englishReading[1] as BitSet)
char[] labels = englishReading[2] as char[]
int[] termIds = englishReading[3] as int[]
writeArray(output, labels.length, Character.BYTES) { it.asCharBuffer().put(labels) }
writeLabels(output, labels)
writeArray(output, termIds.length, Integer.BYTES) { it.asIntBuffer().put(termIds) }
}
writeEntries("word.compact", 0x4B455731) { output -> // KEW1
writeEntries("word.compact", 0x4B455732) { output -> // KEW2
writeBitSet(output, englishWord[0] as BitSet)
writeBitSet(output, englishWord[1] as BitSet)
char[] labels = englishWord[2] as char[]
writeArray(output, labels.length, Character.BYTES) { it.asCharBuffer().put(labels) }
writeLabels(output, labels)
}
writeEntries("token.compact", 0x4B455431) { output -> // KET1
short[] costs = englishToken[0] as short[]
Expand Down Expand Up @@ -1325,6 +1366,7 @@ tasks.named("preBuild").configure {
}

dependencies {
implementation 'org.tukaani:xz:1.12'
implementation 'androidx.profileinstaller:profileinstaller:1.4.0'
implementation 'androidx.appcompat:appcompat:1.7.1'
implementation 'com.google.android.material:material:1.13.0'
Expand Down
33 changes: 33 additions & 0 deletions app/src/main/assets/software-licenses/catalog.json
Original file line number Diff line number Diff line change
Expand Up @@ -3686,6 +3686,39 @@
"liteFdroidRelease"
],
"notices": []
},
{
"group": "org.tukaani",
"name": "xz",
"version": "1.12",
"pomUrl": "https://repo.maven.apache.org/maven2/org/tukaani/xz/1.12/xz-1.12.pom",
"licenses": [
{
"name": "0BSD"
}
],
"evidence": [
{
"url": "https://repo.maven.apache.org/maven2/org/tukaani/xz/1.12/xz-1.12.pom",
"sha256": "937b44d869c059c39f07ae194d8cd6219f5aebf2200ef9707f0b50b7837da281"
},
{
"url": "https://raw.githubusercontent.com/tukaani-project/xz-java/v1.12/COPYING",
"sha256": "1e77ef26cee5216d6531e49cbf2f6111a0270dc7b322fc65477063c5aff88766"
}
],
"variants": [
"fullStandardRelease",
"liteStandardRelease",
"liteFdroidRelease"
],
"notices": [
{
"title": "XZ for Java 1.12 COPYING (0BSD)",
"asset": "software-licenses/xz-1.12-0bsd.txt",
"sha256": "1e77ef26cee5216d6531e49cbf2f6111a0270dc7b322fc65477063c5aff88766"
}
]
}
]
}
13 changes: 13 additions & 0 deletions app/src/main/assets/software-licenses/xz-1.12-0bsd.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
Copyright (C) The XZ for Java authors and contributors

Permission to use, copy, modify, and/or distribute this
software for any purpose with or without fee is hereby granted.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,15 @@ internal object BundledEnglishDictionaryReader {
reading: LOUDSWithTermId,
word: LOUDS,
token: TokenArray,
): CompactEnglishDictionary = readIndex(map(descriptor), reading, word, token)

internal fun readIndex(
source: ByteBuffer,
reading: LOUDSWithTermId,
word: LOUDS,
token: TokenArray,
): CompactEnglishDictionary {
val input = Cursor(descriptor)
val input = Cursor(source)
require(input.readInt() == MAGIC) { "Invalid English lookup index magic" }
// The build indexes the used words of each BitSet. A serialized BitSet may
// also retain spare words that would make its runtime rank table larger.
Expand Down Expand Up @@ -67,39 +74,50 @@ internal object BundledEnglishDictionaryReader {
if (size() == (length() + Long.SIZE_BITS - 1) / Long.SIZE_BITS * Long.SIZE_BITS) this
else BitSet.valueOf(toLongArray())

fun readReading(descriptor: AssetFileDescriptor): LOUDSWithTermId = read(descriptor, 0x4B455231) {
fun readReading(descriptor: AssetFileDescriptor): LOUDSWithTermId = readReading(map(descriptor))

internal fun readReading(source: ByteBuffer): LOUDSWithTermId = read(source, 0x4B455232) {
val lbs = readBitSet()
val leaves = readBitSet()
LOUDSWithTermId(lbs, readCharArray(), leaves, readIntArray())
LOUDSWithTermId(lbs, readLabels(), leaves, readIntArray())
}

fun readWord(descriptor: AssetFileDescriptor): LOUDS = read(descriptor, 0x4B455731) {
fun readWord(descriptor: AssetFileDescriptor): LOUDS = readWord(map(descriptor))

internal fun readWord(source: ByteBuffer): LOUDS = read(source, 0x4B455732) {
val lbs = readBitSet()
val leaves = readBitSet()
LOUDS(lbs, readCharArray(), leaves)
LOUDS(lbs, readLabels(), leaves)
}

fun readToken(descriptor: AssetFileDescriptor): TokenArray = read(descriptor, 0x4B455431) {
fun readToken(descriptor: AssetFileDescriptor): TokenArray = readToken(map(descriptor))

internal fun readToken(source: ByteBuffer): TokenArray = read(source, 0x4B455431) {
TokenArray.fromArrays(readShortArray(), readIntArray(), readBitSet())
}

private inline fun <T> read(
descriptor: AssetFileDescriptor,
source: ByteBuffer,
magic: Int,
decode: Cursor.() -> T,
): T {
val input = Cursor(descriptor)
val input = Cursor(source)
require(input.readInt() == magic) { "Invalid bundled English dictionary header" }
return input.decode().also {
require(input.remaining == 0) { "Unexpected trailing English dictionary data" }
}
}

private class Cursor(descriptor: AssetFileDescriptor) {
private val buffer: ByteBuffer = FileInputStream(descriptor.fileDescriptor).channel.use { channel ->
private fun map(descriptor: AssetFileDescriptor): ByteBuffer =
FileInputStream(descriptor.fileDescriptor).channel.use { channel ->
require(descriptor.length > 0L) { "English dictionary asset must have a known length" }
channel.map(FileChannel.MapMode.READ_ONLY, descriptor.startOffset, descriptor.length)
}.order(ByteOrder.BIG_ENDIAN)
}

private class Cursor(source: ByteBuffer) {
constructor(descriptor: AssetFileDescriptor) : this(map(descriptor))

private val buffer = source.duplicate().order(ByteOrder.BIG_ENDIAN)

val remaining: Int
get() = buffer.remaining()
Expand Down Expand Up @@ -130,6 +148,18 @@ internal object BundledEnglishDictionaryReader {
return CharArray(size).also { take(size * Char.SIZE_BYTES).asCharBuffer().get(it) }
}

fun readLabels(): CharArray {
val alphabet = readCharArray()
require(alphabet.size <= 256) { "Invalid English label alphabet" }
if (alphabet.isEmpty()) return readCharArray()
val size = readSizedCount(Byte.SIZE_BYTES)
return CharArray(size) {
val index = buffer.get().toInt() and 0xff
require(index < alphabet.size) { "Invalid English label index: $index" }
alphabet[index]
}
}

fun readShortArray(): ShortArray {
val size = readSizedCount(Short.SIZE_BYTES)
return ShortArray(size).also { take(size * Short.SIZE_BYTES).asShortBuffer().get(it) }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,8 @@ internal class RuntimeGrammarRoles private constructor(

internal fun read(source: ByteBuffer): RuntimeGrammarRoles = read(source, validateAllRecords = true)

internal fun readBundled(source: ByteBuffer): RuntimeGrammarRoles = read(source, validateAllRecords = false)

private fun read(source: ByteBuffer, validateAllRecords: Boolean): RuntimeGrammarRoles {
val data = source.slice().asReadOnlyBuffer().order(ByteOrder.BIG_ENDIAN)
require(data.capacity() in HEADER_SIZE..MAX_INDEX_SIZE && data.getInt(0) == MAGIC &&
Expand Down
Loading
Loading