From 5583ad203fff84f637978367b1e5643eeaa28b56 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 18:47:31 -0400 Subject: [PATCH 01/10] Add Java JNI backend (procs and consts on primitives and strings) -d:gennyJava generates a C file against jni.h with a JNI_OnLoad that calls NimMain and RegisterNatives, plus a Java class with static native methods and constants. Objects, ref objects, seqs and enums are skipped with a compile-time warning for now. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/java.yml | 24 ++++ .gitignore | 3 + README.md | 1 + src/genny.nim | 10 +- src/genny/languages/java.nim | 212 +++++++++++++++++++++++++++++++++++ tests/TestJavaMain.java | 32 ++++++ tests/test_java.nim | 59 ++++++++++ 7 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/java.yml create mode 100644 src/genny/languages/java.nim create mode 100644 tests/TestJavaMain.java create mode 100644 tests/test_java.nim diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml new file mode 100644 index 0000000..356a7c1 --- /dev/null +++ b/.github/workflows/java.yml @@ -0,0 +1,24 @@ +name: Java +on: + push: + branches: [master] + pull_request: + branches: [master] +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: treeform/setup-nim-action@v6 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + - name: Build standard test DLL with the Java backend + run: nim c --mm:arc --app:lib -d:gennyJava -o:tests/generated/libtest.so tests/test.nim + - name: Build Java test DLL + run: nim c --mm:arc --app:lib -d:gennyJava -o:tests/generated_java/libtest_java.so tests/test_java.nim + - name: Compile Java test + run: javac -d tests/generated_java/classes tests/generated_java/java/genny/TestJava.java tests/TestJavaMain.java + - name: Run Java test + run: java -cp tests/generated_java/classes TestJavaMain tests/generated_java/libtest_java.so diff --git a/.gitignore b/.gitignore index 55b8f35..3d26540 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ tests/generated/*_native.c .* !.github/ !.github/** +tests/generated/*_jni.c +tests/generated/java/ +tests/generated_java/ diff --git a/README.md b/README.md index aa11c13..5b055b9 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Nim | {.importc.} | ✅ | ✅ | ✅ | ✅ | ✅ Python | ctypes | ✅ | ✅ | ✅ | ✅ | ✅ | Node.js | ffi-napi | ✅ | ✅ | ✅ | ✅ | no | C | .h | ✅ | ✅ | ✅ | ✅ | no | +Java | JNI | no | no | no | no | no | ## Quest for "nice" language binding. diff --git a/src/genny.nim b/src/genny.nim index 63c0649..3df82d0 100644 --- a/src/genny.nim +++ b/src/genny.nim @@ -7,9 +7,10 @@ when defined(gennyNode): import genny/languages/node when defined(gennyPython): import genny/languages/python when defined(gennyPythonNative): import genny/languages/python_native when defined(gennyZig): import genny/languages/zig +when defined(gennyJava): import genny/languages/java -when not defined(gennyNim) and not defined(gennyPython) and not defined(gennyPythonNative) and not defined(gennyNode) and not defined(gennyC) and not defined(gennyCpp) and not defined(gennyZig): - {.error: "Please define one of the genny languages. Use -d:gennyNim, -d:gennyPython, -d:gennyPythonNative, -d:gennyNode, -d:gennyC, -d:gennyCpp, -d:gennyZig to define the languages you want to export.".} +when not defined(gennyNim) and not defined(gennyPython) and not defined(gennyPythonNative) and not defined(gennyNode) and not defined(gennyC) and not defined(gennyCpp) and not defined(gennyZig) and not defined(gennyJava): + {.error: "Please define one of the genny languages. Use -d:gennyNim, -d:gennyPython, -d:gennyPythonNative, -d:gennyNode, -d:gennyC, -d:gennyCpp, -d:gennyZig, -d:gennyJava to define the languages you want to export.".} template discard2(f: untyped): untyped = when(compiles do: discard f): @@ -49,6 +50,7 @@ macro exportConstsTyped(body: typed) = when defined(gennyC): exportConstC(sym) when defined(gennyCpp): exportConstCpp(sym) when defined(gennyZig): exportConstZig(sym) + when defined(gennyJava): exportConstJava(sym) template exportConsts*(body: untyped) = ## Exports a list of constants. @@ -133,6 +135,7 @@ proc procTyped( when defined(gennyC): exportProcC(procSym, owner, prefixes) when defined(gennyCpp): exportProcCpp(procSym, owner, prefixes) when defined(gennyZig): exportProcZig(procSym, owner, prefixes) + when defined(gennyJava): exportProcJava(procSym, owner, prefixes) macro exportProcsUntyped(body: untyped) = result = newNimNode(nnkStmtList) @@ -216,6 +219,7 @@ macro exportObjectTyped(body: typed) = when defined(gennyC): exportProcC(procSym, sym, prefixes) when defined(gennyCpp): exportProcCpp(procSym, sym, prefixes) when defined(gennyZig): exportProcZig(procSym, sym, prefixes) + when defined(gennyJava): exportProcJava(procSym, sym, prefixes) when defined(gennyZig): exportCloseObjectZig() when defined(gennyCpp): exportCloseObjectCpp() @@ -357,6 +361,7 @@ macro exportRefObjectTyped(body: typed) = when defined(gennyC): exportProcC(procSym, sym, prefixes) when defined(gennyCpp): exportProcCpp(procSym, sym, prefixes) when defined(gennyZig): exportProcZig(procSym, sym, prefixes) + when defined(gennyJava): exportProcJava(procSym, sym, prefixes) when defined(gennyCpp): exportCloseObjectCpp() when defined(gennyZig): exportCloseObjectZig() @@ -380,3 +385,4 @@ macro writeFiles*(dir, lib: static[string]) = when defined(gennyC): writeC(dir, lib) when defined(gennyCpp): writeCpp(dir, lib) when defined(gennyZig): writeZig(dir, lib) + when defined(gennyJava): result.add writeJava(dir, lib) diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim new file mode 100644 index 0000000..b11b9f0 --- /dev/null +++ b/src/genny/languages/java.nim @@ -0,0 +1,212 @@ +import + std/[macros, os, strformat, strutils], + ../common + +const gennyJavaPackage {.strdefine.} = "genny" + +type JavaType = object + java, jni, sig, c: string + +var + javaMembers {.compiletime.}: string + cProtos {.compiletime.}: string + cWrappers {.compiletime.}: string + cMethods {.compiletime.}: string + +proc stripSink(sym: NimNode): NimNode = + if sym.kind == nnkBracketExpr and sym[0].repr == "sink": + sym[1] + else: + sym + +proc isVoid(sym: NimNode): bool = + sym.kind == nnkEmpty or sym.repr == "void" + +proc javaType(sym: NimNode, supported: var bool): JavaType = + supported = true + let typ = sym.stripSink + if typ.kind == nnkBracketExpr: + supported = false + return + case typ.repr + of "int", "int64", "uint", "uint64": + JavaType(java: "long", jni: "jlong", sig: "J", c: "int64_t") + of "int32", "uint32": + JavaType(java: "int", jni: "jint", sig: "I", c: "int32_t") + of "int16", "uint16": + JavaType(java: "short", jni: "jshort", sig: "S", c: "int16_t") + of "int8", "uint8", "byte": + JavaType(java: "byte", jni: "jbyte", sig: "B", c: "int8_t") + of "bool": + JavaType(java: "boolean", jni: "jboolean", sig: "Z", c: "bool") + of "float32": + JavaType(java: "float", jni: "jfloat", sig: "F", c: "float") + of "float", "float64": + JavaType(java: "double", jni: "jdouble", sig: "D", c: "double") + of "string": + JavaType(java: "String", jni: "jstring", sig: "Ljava/lang/String;", c: "char *") + else: + supported = false + JavaType() + +proc javaClassName(lib: string): string = + toCamelCase(toSnakeCase(lib)) + +proc exportConstJava*(sym: NimNode) = + var supported: bool + let t = javaType(sym.getTypeInst(), supported) + if not supported: + warning(&"genny java: skipping const {sym.repr}, unsupported type", sym) + return + var valueNode = sym.getImpl()[2] + while valueNode.kind == nnkSym: + valueNode = valueNode.getImpl()[2] + var value = valueNode.repr + case t.java + of "long": value.add "L" + of "float": value.add "f" + of "short", "byte": value = &"({t.java}){value}" + else: discard + javaMembers.add &" public static final {t.java} {toCapSnakeCase(sym.repr)} = {value};\n" + +proc exportProcJava*( + sym: NimNode, + owner: NimNode = nil, + prefixes: openarray[NimNode] = [] +) = + if owner != nil: + warning(&"genny java: skipping {owner.repr}.{sym.repr}, object procs not supported yet", sym) + return + + let + procType = sym.getTypeInst() + procParams = procType[0][1 .. ^1] + procReturn = procType[0][0] + apiName = "$lib_" & toSnakeCase(sym.repr) + wrapperName = "genny_jni_" & toSnakeCase(sym.repr) + + var + supported = true + ok: bool + params: seq[(string, JavaType)] + for param in procParams: + for i in 0 .. param.len - 3: + let t = javaType(param[^2], ok) + supported = supported and ok + params.add((toSnakeCase(param[i].repr), t)) + var ret = JavaType(java: "void", jni: "void", sig: "V", c: "void") + if not procReturn.isVoid: + ret = javaType(procReturn, ok) + supported = supported and ok + if not supported: + warning(&"genny java: skipping proc {sym.repr}, unsupported types", sym) + return + + var + javaParams, cProtoParams, jniParams, sig, callArgs, setup, cleanup: seq[string] + for (name, t) in params: + javaParams.add &"{t.java} {toVarCase(toCamelCase(name))}" + cProtoParams.add &"{t.c} {name}" + jniParams.add &"{t.jni} {name}" + sig.add t.sig + if t.java == "String": + setup.add &" const char *{name}_c = (*env)->GetStringUTFChars(env, {name}, NULL);\n" + setup.add &" if ({name}_c == NULL) return{(if ret.java == \"void\": \"\" else: \" 0\")};\n" + cleanup.add &" (*env)->ReleaseStringUTFChars(env, {name}, {name}_c);\n" + callArgs.add &"(char *){name}_c" + elif t.java == "boolean": + callArgs.add &"{name} == JNI_TRUE" + else: + callArgs.add &"({t.c}){name}" + + javaMembers.add &" public static native {ret.java} {sym.repr}({javaParams.join(\", \")});\n" + + cProtos.add &"extern {ret.c} {apiName}({cProtoParams.join(\", \")});\n" + + var allJniParams = @["JNIEnv *env", "jclass cls"] + allJniParams.add jniParams + cWrappers.add &"static {ret.jni} {wrapperName}({allJniParams.join(\", \")}) {{\n" + cWrappers.add " (void)cls;\n" + if params.len == 0 or setup.len == 0: + cWrappers.add " (void)env;\n" + cWrappers.add setup.join("") + let call = &"{apiName}({callArgs.join(\", \")})" + case ret.java + of "void": + cWrappers.add &" {call};\n" + cWrappers.add cleanup.join("") + of "String": + cWrappers.add &" {ret.c}genny_result = {call};\n" + cWrappers.add cleanup.join("") + cWrappers.add " return (*env)->NewStringUTF(env, genny_result == NULL ? \"\" : genny_result);\n" + of "boolean": + cWrappers.add &" {ret.jni} genny_result = {call} ? JNI_TRUE : JNI_FALSE;\n" + cWrappers.add cleanup.join("") + cWrappers.add " return genny_result;\n" + else: + cWrappers.add &" {ret.jni} genny_result = ({ret.jni}){call};\n" + cWrappers.add cleanup.join("") + cWrappers.add " return genny_result;\n" + cWrappers.add "}\n\n" + + cMethods.add &" {{\"{sym.repr}\", \"({sig.join(\"\")}){ret.sig}\", (void *){wrapperName}}},\n" + +proc writeJava*(dir, lib: string): NimNode = + let + className = javaClassName(lib) + classPath = gennyJavaPackage.replace('.', '/') & "/" & className + javaDir = dir / "java" / gennyJavaPackage.replace('.', '/') + cPath = dir / (toSnakeCase(lib) & "_jni.c") + javaHome = getEnv("JAVA_HOME") + + if javaHome == "": + error("genny java: JAVA_HOME must be set to a JDK so jni.h can be found") + + createDir(javaDir) + + var java = "" + java.add &"package {gennyJavaPackage};\n\n" + java.add &"public final class {className} {{\n" + java.add &" private {className}() {{}}\n\n" + java.add " /** Loads the native library from an absolute path. */\n" + java.add " public static void load(String absolutePath) {\n" + java.add " System.load(absolutePath);\n" + java.add " }\n\n" + java.add javaMembers + java.add "}\n" + writeFile(javaDir / (className & ".java"), java) + + var code = "" + code.add "#include \n" + code.add "#include \n" + code.add "#include \n\n" + code.add "extern void NimMain(void);\n\n" + code.add cProtos + code.add "\n" + code.add cWrappers + code.add "static JNINativeMethod genny_jni_methods[] = {\n" + code.add cMethods + code.add "};\n\n" + code.add "JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {\n" + code.add " (void)reserved;\n" + code.add " JNIEnv *env;\n" + code.add " if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_8) != JNI_OK) return JNI_ERR;\n" + code.add " NimMain();\n" + code.add &" jclass cls = (*env)->FindClass(env, \"{classPath}\");\n" + code.add " if (cls == NULL) return JNI_ERR;\n" + code.add " jint count = (jint)(sizeof(genny_jni_methods) / sizeof(genny_jni_methods[0]));\n" + code.add " if (count > 0 && (*env)->RegisterNatives(env, cls, genny_jni_methods, count) != 0) return JNI_ERR;\n" + code.add " return JNI_VERSION_1_8;\n" + code.add "}\n" + code = code.replace("$lib", toSnakeCase(lib)) + writeFile(cPath, code) + + result = newStmtList() + let + includeOpt = "-I" & (javaHome / "include") + includePlatformOpt = "-I" & (javaHome / "include" / "linux") + compilePath = cPath + result.add quote do: + {.passC: `includeOpt`.} + {.passC: `includePlatformOpt`.} + {.compile: `compilePath`.} diff --git a/tests/TestJavaMain.java b/tests/TestJavaMain.java new file mode 100644 index 0000000..f0051e2 --- /dev/null +++ b/tests/TestJavaMain.java @@ -0,0 +1,32 @@ +import genny.TestJava; + +public class TestJavaMain { + static void check(boolean ok, String what) { + if (!ok) throw new AssertionError(what); + System.out.println("ok " + what); + } + + public static void main(String[] args) { + TestJava.load(new java.io.File(args[0]).getAbsolutePath()); + + check(TestJava.JAVA_LONG_CONST == 123L, "long const"); + check(TestJava.JAVA_DOUBLE_CONST == 1.5, "double const"); + check(TestJava.JAVA_STRING_CONST.equals("genny"), "string const"); + + check(TestJava.addInts(40, 2) == 42, "addInts"); + check(TestJava.addInts(Long.MAX_VALUE - 1, 1) == Long.MAX_VALUE, "addInts 64-bit"); + check(TestJava.addInt32s(1, 2) == 3, "addInt32s"); + check(TestJava.mulFloats(1.5f, 2f) == 3f, "mulFloats"); + check(TestJava.mulDoubles(1.5, 2) == 3.0, "mulDoubles"); + check(TestJava.negate(true) == false, "negate true"); + check(TestJava.negate(false) == true, "negate false"); + check(TestJava.echoByte((byte)7) == 7, "echoByte"); + check(TestJava.greet("Java").equals("Hello, Java!"), "greet"); + check(TestJava.helloWorld().equals("Hello world from Nim!"), "helloWorld"); + + for (int i = 0; i < 3; i++) TestJava.bump(); + check(TestJava.getCallCount() == 3, "void proc + state"); + + System.out.println("All Java tests passed"); + } +} diff --git a/tests/test_java.nim b/tests/test_java.nim new file mode 100644 index 0000000..6040dc4 --- /dev/null +++ b/tests/test_java.nim @@ -0,0 +1,59 @@ +import genny + +const + javaLongConst = 123 + javaDoubleConst = 1.5 + javaStringConst = "genny" + +exportConsts: + javaLongConst + javaDoubleConst + javaStringConst + +proc addInts(a, b: int): int = + a + b + +proc addInt32s(a, b: int32): int32 = + a + b + +proc mulFloats(a, b: float32): float32 = + a * b + +proc mulDoubles(a, b: float): float = + a * b + +proc negate(a: bool): bool = + not a + +proc echoByte(a: byte): byte = + a + +proc greet(name: string): string = + "Hello, " & name & "!" + +proc helloWorld(): string = + "Hello world from Nim!" + +var callCount: int + +proc bump() = + inc callCount + +proc getCallCount(): int = + callCount + +exportProcs: + addInts + addInt32s + mulFloats + mulDoubles + negate + echoByte + greet + helloWorld + bump + getCallCount + +writeFiles("tests/generated_java", "testJava") + +include generated_java/internal From 6aedd4dc2509472ae09184794ad940f629cc7698 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 18:49:27 -0400 Subject: [PATCH 02/10] Require -d:noSignalHandler for the Java backend The JVM raises SIGSEGV on purpose for null checks and safepoints. Nim's signal handler catches those and exits, killing the host JVM. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/java.yml | 4 ++-- src/genny/languages/java.nim | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 356a7c1..d9494e8 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -15,9 +15,9 @@ jobs: distribution: temurin java-version: '17' - name: Build standard test DLL with the Java backend - run: nim c --mm:arc --app:lib -d:gennyJava -o:tests/generated/libtest.so tests/test.nim + run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated/libtest.so tests/test.nim - name: Build Java test DLL - run: nim c --mm:arc --app:lib -d:gennyJava -o:tests/generated_java/libtest_java.so tests/test_java.nim + run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated_java/libtest_java.so tests/test_java.nim - name: Compile Java test run: javac -d tests/generated_java/classes tests/generated_java/java/genny/TestJava.java tests/TestJavaMain.java - name: Run Java test diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index b11b9f0..f2073ae 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -4,6 +4,11 @@ import const gennyJavaPackage {.strdefine.} = "genny" +when not defined(noSignalHandler): + # The JVM raises SIGSEGV on purpose (null checks, safepoints) and handles + # it itself; Nim's handler would turn those into a process exit. + {.error: "genny java: compile with -d:noSignalHandler, Nim's SIGSEGV handler breaks the JVM".} + type JavaType = object java, jni, sig, c: string From 82acbe82a7df4d3854c7758043e20b51806484c2 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 18:55:23 -0400 Subject: [PATCH 03/10] Note the string-return lifetime quirk in the Java backend Co-Authored-By: Claude Opus 5.5 --- src/genny/languages/java.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index f2073ae..f2448b6 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -141,6 +141,9 @@ proc exportProcJava*( cWrappers.add &" {call};\n" cWrappers.add cleanup.join("") of "String": + # TODO: under ARC/ORC the char* from the internal export points into a + # Nim string that is already freed when it returns. NewStringUTF must copy + # it before anything else allocates; revisit with a threadvar holder. cWrappers.add &" {ret.c}genny_result = {call};\n" cWrappers.add cleanup.join("") cWrappers.add " return (*env)->NewStringUTF(env, genny_result == NULL ? \"\" : genny_result);\n" From 1e2d5f1f2e55d935b3231ef828e207cd88e1c375 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 19:28:36 -0400 Subject: [PATCH 04/10] Java backend: enums, objects, ref objects, seqs and errors Natives are registered on the library class and wrapped by a public API: enums become Java enums, value objects become classes converted field by field in C (cached class, field and constructor IDs), ref objects and seqs become AutoCloseable handle classes released by a Cleaner, procs on objects become instance methods, and raising procs throw Exception. TestJavaStandard runs the standard test library through Java. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/java.yml | 6 +- .gitignore | 1 + README.md | 2 +- src/genny.nim | 4 + src/genny/languages/java.nim | 650 ++++++++++++++++++++++++++++++----- tests/TestJavaMain.java | 36 +- tests/TestJavaStandard.java | 59 ++++ tests/test_java.nim | 85 ++++- 8 files changed, 746 insertions(+), 97 deletions(-) create mode 100644 tests/TestJavaStandard.java diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index d9494e8..b44bff3 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -16,9 +16,13 @@ jobs: java-version: '17' - name: Build standard test DLL with the Java backend run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated/libtest.so tests/test.nim + - name: Run Java test against the standard test DLL + run: | + javac -d tests/generated/classes tests/generated/java/genny/*.java tests/TestJavaStandard.java + java -cp tests/generated/classes TestJavaStandard tests/generated/libtest.so - name: Build Java test DLL run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated_java/libtest_java.so tests/test_java.nim - name: Compile Java test - run: javac -d tests/generated_java/classes tests/generated_java/java/genny/TestJava.java tests/TestJavaMain.java + run: javac -d tests/generated_java/classes tests/generated_java/java/genny/*.java tests/TestJavaMain.java - name: Run Java test run: java -cp tests/generated_java/classes TestJavaMain tests/generated_java/libtest_java.so diff --git a/.gitignore b/.gitignore index 3d26540..6869b11 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ tests/generated/*_native.c tests/generated/*_jni.c tests/generated/java/ tests/generated_java/ +*.class diff --git a/README.md b/README.md index 5b055b9..fcad3c9 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Nim | {.importc.} | ✅ | ✅ | ✅ | ✅ | ✅ Python | ctypes | ✅ | ✅ | ✅ | ✅ | ✅ | Node.js | ffi-napi | ✅ | ✅ | ✅ | ✅ | no | C | .h | ✅ | ✅ | ✅ | ✅ | no | -Java | JNI | no | no | no | no | no | +Java | JNI | ✅ | ✅ | ✅ | ✅ | ✅ | ## Quest for "nice" language binding. diff --git a/src/genny.nim b/src/genny.nim index 3df82d0..97d0123 100644 --- a/src/genny.nim +++ b/src/genny.nim @@ -76,6 +76,7 @@ macro exportEnumsTyped(body: typed) = when defined(gennyC): exportEnumC(sym) when defined(gennyCpp): exportEnumCpp(sym) when defined(gennyZig): exportEnumZig(sym) + when defined(gennyJava): exportEnumJava(sym) template exportEnums*(body: untyped) = ## Exports a list of enums. @@ -198,6 +199,7 @@ macro exportObjectTyped(body: typed) = when defined(gennyC): exportObjectC(sym, constructor) when defined(gennyCpp): exportObjectCpp(sym, constructor) when defined(gennyZig): exportObjectZig(sym, constructor) + when defined(gennyJava): exportObjectJava(sym, constructor) if procsBlock[1].len > 0: var procsSeen: seq[string] @@ -260,6 +262,7 @@ macro exportSeqTyped(body: typed) = when defined(gennyC): exportSeqC(sym) when defined(gennyCpp): exportSeqCpp(sym) when defined(gennyZig): exportSeqZig(sym) + when defined(gennyJava): exportSeqJava(sym) for entry in body.asStmtList()[1 .. ^1]: procTyped(entry, sym) @@ -340,6 +343,7 @@ macro exportRefObjectTyped(body: typed) = when defined(gennyC): exportRefObjectC(sym, fields, constructor) when defined(gennyCpp): exportRefObjectCpp(sym, fields, constructor) when defined(gennyZig): exportRefObjectZig(sym, fields, constructor) + when defined(gennyJava): exportRefObjectJava(sym, fields, constructor) if procsBlock[1].len > 0: var procsSeen: seq[string] diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index f2448b6..1512c19 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -1,5 +1,5 @@ import - std/[macros, os, strformat, strutils], + std/[macros, os, sets, strformat, strutils, tables], ../common const gennyJavaPackage {.strdefine.} = "genny" @@ -9,14 +9,39 @@ when not defined(noSignalHandler): # it itself; Nim's handler would turn those into a process exit. {.error: "genny java: compile with -d:noSignalHandler, Nim's SIGSEGV handler breaks the JVM".} -type JavaType = object - java, jni, sig, c: string +type + Kind = enum + kVoid, kPrim, kString, kEnum, kValue, kRef, kUnsupported + + JType = object + kind: Kind + java: string ## Type in the public Java API. + native: string ## Type in the `native` declaration. + jni: string ## C type of the JNI value. + sig: string ## JNI signature of `native`. + publicSig: string ## JNI signature of `java` (value object fields). + c: string ## C type of the Nim export. + field: string ## GetField / New suffix for primitives. + +const libClass = "$LibClass" var - javaMembers {.compiletime.}: string + libMembers {.compiletime.}: string + classMembers {.compiletime.}: OrderedTable[string, string] + classKinds {.compiletime.}: Table[string, Kind] + enumNames {.compiletime.}: HashSet[string] + enumSizes {.compiletime.}: Table[string, int] + enumValues {.compiletime.}: Table[string, seq[string]] + valueNames {.compiletime.}: HashSet[string] + valueFields {.compiletime.}: Table[string, seq[(string, NimNode)]] + refNames {.compiletime.}: HashSet[string] + cTypes {.compiletime.}: string cProtos {.compiletime.}: string + cHelpers {.compiletime.}: string cWrappers {.compiletime.}: string cMethods {.compiletime.}: string + cOnLoad {.compiletime.}: string + needsErrorBridge {.compiletime.}: bool proc stripSink(sym: NimNode): NimNode = if sym.kind == nnkBracketExpr and sym[0].repr == "sink": @@ -25,42 +50,175 @@ proc stripSink(sym: NimNode): NimNode = sym proc isVoid(sym: NimNode): bool = - sym.kind == nnkEmpty or sym.repr == "void" + sym.kind == nnkEmpty or sym.repr in ["void", "", "nil"] + +proc classPath(name: string): string = + gennyJavaPackage.replace('.', '/') & "/" & name -proc javaType(sym: NimNode, supported: var bool): JavaType = - supported = true +proc prim(java, jni, sig, c, field: string): JType = + JType(kind: kPrim, java: java, native: java, jni: jni, sig: sig, + publicSig: sig, c: c, field: field) + +proc jtype(sym: NimNode): JType = let typ = sym.stripSink + if typ.isVoid: + return JType(kind: kVoid, java: "void", native: "void", jni: "void", + sig: "V", c: "void") + let name = typ.getName() if typ.kind == nnkBracketExpr: - supported = false - return - case typ.repr + if typ[0].repr == "seq" and name in refNames: + return JType(kind: kRef, java: name, native: "long", jni: "jlong", + sig: "J", publicSig: &"L{classPath(name)};", c: "void *") + return JType(kind: kUnsupported) + if name in enumNames: + let c = if enumSizes[name] <= 1: "uint8_t" else: "uint16_t" + return JType(kind: kEnum, java: name, native: "int", jni: "jint", + sig: "I", publicSig: &"L{classPath(name)};", c: c) + if name in valueNames: + return JType(kind: kValue, java: name, native: name, jni: "jobject", + sig: &"L{classPath(name)};", publicSig: &"L{classPath(name)};", c: name) + if name in refNames: + return JType(kind: kRef, java: name, native: "long", jni: "jlong", + sig: "J", publicSig: &"L{classPath(name)};", c: "void *") + case name of "int", "int64", "uint", "uint64": - JavaType(java: "long", jni: "jlong", sig: "J", c: "int64_t") + prim("long", "jlong", "J", "int64_t", "Long") of "int32", "uint32": - JavaType(java: "int", jni: "jint", sig: "I", c: "int32_t") + prim("int", "jint", "I", "int32_t", "Int") of "int16", "uint16": - JavaType(java: "short", jni: "jshort", sig: "S", c: "int16_t") + prim("short", "jshort", "S", "int16_t", "Short") of "int8", "uint8", "byte": - JavaType(java: "byte", jni: "jbyte", sig: "B", c: "int8_t") + prim("byte", "jbyte", "B", "int8_t", "Byte") of "bool": - JavaType(java: "boolean", jni: "jboolean", sig: "Z", c: "bool") + prim("boolean", "jboolean", "Z", "bool", "Boolean") of "float32": - JavaType(java: "float", jni: "jfloat", sig: "F", c: "float") + prim("float", "jfloat", "F", "float", "Float") of "float", "float64": - JavaType(java: "double", jni: "jdouble", sig: "D", c: "double") + prim("double", "jdouble", "D", "double", "Double") of "string": - JavaType(java: "String", jni: "jstring", sig: "Ljava/lang/String;", c: "char *") + JType(kind: kString, java: "String", native: "String", jni: "jstring", + sig: "Ljava/lang/String;", publicSig: "Ljava/lang/String;", c: "char *") else: - supported = false - JavaType() + JType(kind: kUnsupported) + +proc javaToNative(t: JType, expr: string): string = + ## Converts a public Java value to the value passed to `native`. + case t.kind + of kEnum: &"{expr}.ordinal()" + of kRef: &"({expr} == null ? 0L : {expr}.handle)" + else: expr + +proc nativeToJava(t: JType, expr: string): string = + ## Converts a `native` return value to the public Java value. + case t.kind + of kEnum: &"{t.java}.fromOrdinal({expr})" + of kRef: &"{t.java}.wrap({expr})" + else: expr + +proc zeroReturn(t: JType): string = + case t.kind + of kVoid: "return;" + of kString, kValue: "return NULL;" + else: "return 0;" + +proc cArg(t: JType, name: string, setup, cleanup: var string): string = + ## Converts a JNI argument to the C value the Nim export takes. + case t.kind + of kString: + setup.add &" const char *{name}_c = {name} == NULL ? \"\" : (*env)->GetStringUTFChars(env, {name}, NULL);\n" + cleanup.add &" if ({name} != NULL) (*env)->ReleaseStringUTFChars(env, {name}, {name}_c);\n" + &"(char *){name}_c" + of kPrim: + if t.java == "boolean": &"{name} == JNI_TRUE" else: &"({t.c}){name}" + of kEnum: &"({t.c}){name}" + of kValue: &"genny_to_{t.c}(env, {name})" + of kRef: &"(void *)(intptr_t){name}" + else: name + +proc cReturn(t: JType, expr: string): string = + ## Converts the C value a Nim export returned to a JNI value. + case t.kind + of kString: + # TODO: under ARC/ORC the char* from the internal export points into a + # Nim string that is already freed when it returns. NewStringUTF must + # copy it before anything else allocates; revisit with a threadvar holder. + &"(*env)->NewStringUTF(env, {expr} == NULL ? \"\" : {expr})" + of kPrim: + if t.java == "boolean": &"({expr} ? JNI_TRUE : JNI_FALSE)" else: &"({t.jni}){expr}" + of kEnum: &"(jint){expr}" + of kValue: &"genny_from_{t.c}(env, {expr})" + of kRef: &"(jlong)(intptr_t){expr}" + else: expr proc javaClassName(lib: string): string = toCamelCase(toSnakeCase(lib)) +proc addMember(className, code: string) = + if className == libClass: + libMembers.add code + else: + classMembers[className] = classMembers.getOrDefault(className) & code + +proc emitNative( + apiName: string, + params: seq[(string, JType)], + ret: JType, + raises: bool +): string = + ## Emits the JNI wrapper for one Nim export and returns the Java native + ## method name, declared on the library class. + let + nativeName = "n_" & apiName.replace("$lib_", "") + wrapperName = "genny_jni_" & apiName.replace("$lib_", "") + + var javaParams, cProtoParams, jniParams, sig, callArgs: seq[string] + var setup, cleanup: string + for i, (name, t) in params: + let pname = &"p{i}" + javaParams.add &"{t.native} {pname}" + cProtoParams.add &"{t.c} {name}" + jniParams.add &"{t.jni} {pname}" + sig.add t.sig + callArgs.add cArg(t, pname, setup, cleanup) + + libMembers.add &" static native {ret.native} {nativeName}({javaParams.join(\", \")});\n" + cProtos.add &"extern {ret.c} {apiName}({cProtoParams.join(\", \")});\n" + + var allJniParams = @["JNIEnv *env", "jclass cls"] + allJniParams.add jniParams + cWrappers.add &"static {ret.jni} {wrapperName}({allJniParams.join(\", \")}) {{\n" + cWrappers.add " (void)env; (void)cls;\n" + cWrappers.add setup + let call = &"{apiName}({callArgs.join(\", \")})" + if ret.kind == kVoid: + cWrappers.add &" {call};\n" + else: + cWrappers.add &" {ret.c} genny_result = {call};\n" + cWrappers.add cleanup + if raises: + needsErrorBridge = true + cWrappers.add " if ($lib_check_error()) {\n" + cWrappers.add " char *genny_error = $lib_take_error();\n" + cWrappers.add " (*env)->ThrowNew(env, genny_exception_class, genny_error == NULL ? \"error\" : genny_error);\n" + cWrappers.add &" {zeroReturn(ret)}\n" + cWrappers.add " }\n" + if ret.kind != kVoid: + cWrappers.add &" return {cReturn(ret, \"genny_result\")};\n" + cWrappers.add "}\n\n" + + cMethods.add &" {{\"{nativeName}\", \"({sig.join(\"\")}){ret.sig}\", (void *){wrapperName}}},\n" + nativeName + +proc javaCall(nativeName: string, args: seq[string], ret: JType): string = + let call = &"{libClass}.{nativeName}({args.join(\", \")})" + if ret.kind == kVoid: + &" {call};\n" + else: + &" return {nativeToJava(ret, call)};\n" + proc exportConstJava*(sym: NimNode) = - var supported: bool - let t = javaType(sym.getTypeInst(), supported) - if not supported: + let t = jtype(sym.getTypeInst()) + if t.kind notin {kPrim, kString}: warning(&"genny java: skipping const {sym.repr}, unsupported type", sym) return var valueNode = sym.getImpl()[2] @@ -72,125 +230,419 @@ proc exportConstJava*(sym: NimNode) = of "float": value.add "f" of "short", "byte": value = &"({t.java}){value}" else: discard - javaMembers.add &" public static final {t.java} {toCapSnakeCase(sym.repr)} = {value};\n" + libMembers.add &" public static final {t.java} {toCapSnakeCase(sym.repr)} = {value};\n" + +proc exportEnumJava*(sym: NimNode) = + let + name = sym.repr + entries = sym.getImpl()[2][1 .. ^1] + var values: seq[string] + for entry in entries: + values.add toCapSnakeCase(entry.repr) + enumNames.incl name + enumSizes[name] = if entries.len <= 256: 1 else: 2 + enumValues[name] = values + classKinds[name] = kEnum proc exportProcJava*( sym: NimNode, owner: NimNode = nil, prefixes: openarray[NimNode] = [] ) = - if owner != nil: - warning(&"genny java: skipping {owner.repr}.{sym.repr}, object procs not supported yet", sym) - return - let procType = sym.getTypeInst() procParams = procType[0][1 .. ^1] procReturn = procType[0][0] - apiName = "$lib_" & toSnakeCase(sym.repr) - wrapperName = "genny_jni_" & toSnakeCase(sym.repr) + ret = jtype(procReturn) + + var apiName = "$lib_" + if owner != nil: + apiName.add &"{toSnakeCase(owner.getName())}_" + for prefix in prefixes: + apiName.add &"{toSnakeCase(prefix.getName())}_" + apiName.add toSnakeCase(sym.repr) var - supported = true - ok: bool - params: seq[(string, JavaType)] + supported = ret.kind != kUnsupported + params: seq[(string, JType)] for param in procParams: for i in 0 .. param.len - 3: - let t = javaType(param[^2], ok) - supported = supported and ok + let t = jtype(param[^2]) + supported = supported and t.kind != kUnsupported params.add((toSnakeCase(param[i].repr), t)) - var ret = JavaType(java: "void", jni: "void", sig: "V", c: "void") - if not procReturn.isVoid: - ret = javaType(procReturn, ok) - supported = supported and ok if not supported: warning(&"genny java: skipping proc {sym.repr}, unsupported types", sym) return + let nativeName = emitNative(apiName, params, ret, sym.raises()) + + let onType = owner != nil and params.len > 0 + var publicParams, args: seq[string] + for i, (name, t) in params: + if i == 0 and onType: + args.add javaToNative(t, "this") + else: + let jname = toVarCase(toCamelCase(name)) + publicParams.add &"{t.java} {jname}" + args.add javaToNative(t, jname) + + let + className = if onType: owner.getName() else: libClass + modifier = if onType: "public" else: "public static" + var code = &" {modifier} {ret.java} {sym.repr}({publicParams.join(\", \")}) {{\n" + code.add javaCall(nativeName, args, ret) + code.add " }\n" + addMember(className, code) + +proc emitValueConversions(name: string, fields: seq[(string, NimNode)]) = + ## C helpers between the Java value class and the Nim struct. + cTypes.add &"typedef struct {name} {{\n" + for (fieldName, fieldType) in fields: + cTypes.add &" {jtype(fieldType).c} {fieldName};\n" + cTypes.add &"}} {name};\n\n" + + cHelpers.add &"static jclass genny_class_{name};\n" + cHelpers.add &"static jmethodID genny_init_{name};\n" + for (fieldName, _) in fields: + cHelpers.add &"static jfieldID genny_field_{name}_{fieldName};\n" + + cHelpers.add &"static {name} genny_to_{name}(JNIEnv *env, jobject o) {{\n" + cHelpers.add &" {name} r;\n" + cHelpers.add " memset(&r, 0, sizeof(r));\n" + cHelpers.add " if (o == NULL) return r;\n" + for (fieldName, fieldType) in fields: + let + t = jtype(fieldType) + fid = &"genny_field_{name}_{fieldName}" + case t.kind + of kPrim: + if t.java == "boolean": + cHelpers.add &" r.{fieldName} = (*env)->GetBooleanField(env, o, {fid}) == JNI_TRUE;\n" + else: + cHelpers.add &" r.{fieldName} = ({t.c})(*env)->Get{t.field}Field(env, o, {fid});\n" + of kEnum: + cHelpers.add &" {{ jobject e = (*env)->GetObjectField(env, o, {fid});\n" + cHelpers.add &" r.{fieldName} = e == NULL ? 0 : ({t.c})(*env)->CallIntMethod(env, e, genny_enum_ordinal);\n" + cHelpers.add " if (e != NULL) (*env)->DeleteLocalRef(env, e); }\n" + of kValue: + cHelpers.add &" {{ jobject v = (*env)->GetObjectField(env, o, {fid});\n" + cHelpers.add &" r.{fieldName} = genny_to_{t.c}(env, v);\n" + cHelpers.add " if (v != NULL) (*env)->DeleteLocalRef(env, v); }\n" + else: discard + cHelpers.add " return r;\n}\n\n" + + var ctorSig, ctorArgs, setup, cleanup: string + for (fieldName, fieldType) in fields: + let t = jtype(fieldType) + ctorSig.add t.publicSig + case t.kind + of kPrim: + if t.java == "boolean": + ctorArgs.add &", (jboolean)(v.{fieldName} ? JNI_TRUE : JNI_FALSE)" + else: + ctorArgs.add &", ({t.jni})v.{fieldName}" + of kEnum: + setup.add &" jobject f_{fieldName} = (*env)->CallStaticObjectMethod(env, genny_class_{t.java}, genny_from_ordinal_{t.java}, (jint)v.{fieldName});\n" + cleanup.add &" (*env)->DeleteLocalRef(env, f_{fieldName});\n" + ctorArgs.add &", f_{fieldName}" + of kValue: + setup.add &" jobject f_{fieldName} = genny_from_{t.c}(env, v.{fieldName});\n" + cleanup.add &" (*env)->DeleteLocalRef(env, f_{fieldName});\n" + ctorArgs.add &", f_{fieldName}" + else: discard + cHelpers.add &"static jobject genny_from_{name}(JNIEnv *env, {name} v) {{\n" + cHelpers.add setup + cHelpers.add &" jobject r = (*env)->NewObject(env, genny_class_{name}, genny_init_{name}{ctorArgs});\n" + cHelpers.add cleanup + cHelpers.add " return r;\n}\n\n" + + cOnLoad.add &" if (!genny_load_class(env, \"{classPath(name)}\", &genny_class_{name})) return JNI_ERR;\n" + cOnLoad.add &" genny_init_{name} = (*env)->GetMethodID(env, genny_class_{name}, \"\", \"({ctorSig})V\");\n" + cOnLoad.add &" if (genny_init_{name} == NULL) return JNI_ERR;\n" + for (fieldName, fieldType) in fields: + let t = jtype(fieldType) + cOnLoad.add &" genny_field_{name}_{fieldName} = (*env)->GetFieldID(env, genny_class_{name}, \"{toVarCase(toCamelCase(fieldName))}\", \"{t.publicSig}\");\n" + cOnLoad.add &" if (genny_field_{name}_{fieldName} == NULL) return JNI_ERR;\n" + +proc exportObjectJava*(sym: NimNode, constructor: NimNode) = + let name = sym.repr var - javaParams, cProtoParams, jniParams, sig, callArgs, setup, cleanup: seq[string] - for (name, t) in params: - javaParams.add &"{t.java} {toVarCase(toCamelCase(name))}" - cProtoParams.add &"{t.c} {name}" - jniParams.add &"{t.jni} {name}" - sig.add t.sig - if t.java == "String": - setup.add &" const char *{name}_c = (*env)->GetStringUTFChars(env, {name}, NULL);\n" - setup.add &" if ({name}_c == NULL) return{(if ret.java == \"void\": \"\" else: \" 0\")};\n" - cleanup.add &" (*env)->ReleaseStringUTFChars(env, {name}, {name}_c);\n" - callArgs.add &"(char *){name}_c" - elif t.java == "boolean": - callArgs.add &"{name} == JNI_TRUE" + fields: seq[(string, NimNode)] + supported = true + for identDefs in sym.getImpl()[2][2]: + for property in identDefs[0 .. ^3]: + let t = jtype(identDefs[^2]) + if t.kind notin {kPrim, kEnum, kValue}: + supported = false + fields.add((toSnakeCase(property[1].repr), identDefs[^2])) + if not supported: + warning(&"genny java: skipping object {name}, only primitive, enum and object fields are supported", sym) + return + + valueNames.incl name + valueFields[name] = fields + classKinds[name] = kValue + emitValueConversions(name, fields) + + var code = "" + var ctorParams, assigns, eqs, hashes: seq[string] + for (fieldName, fieldType) in fields: + let + t = jtype(fieldType) + jname = toVarCase(toCamelCase(fieldName)) + code.add &" public {t.java} {jname};\n" + ctorParams.add &"{t.java} {jname}" + assigns.add &" this.{jname} = {jname};\n" + if t.kind == kPrim: + eqs.add &"{jname} == o.{jname}" else: - callArgs.add &"({t.c}){name}" + eqs.add &"java.util.Objects.equals({jname}, o.{jname})" + hashes.add jname + code.add &"\n public {name}() {{}}\n\n" + code.add &" public {name}({ctorParams.join(\", \")}) {{\n" + code.add assigns.join("") + code.add " }\n\n" + code.add " @Override public boolean equals(Object other) {\n" + code.add &" if (!(other instanceof {name})) return false;\n" + code.add &" {name} o = ({name})other;\n" + code.add &" return {eqs.join(\" && \")};\n" + code.add " }\n\n" + code.add " @Override public int hashCode() {\n" + code.add &" return java.util.Objects.hash({hashes.join(\", \")});\n" + code.add " }\n\n" + addMember(name, code) - javaMembers.add &" public static native {ret.java} {sym.repr}({javaParams.join(\", \")});\n" + if constructor != nil: + let + ctorType = constructor.getTypeInst() + ret = jtype(sym) + var params: seq[(string, JType)] + for param in ctorType[0][1 .. ^1]: + params.add((param[0].repr.split('`')[0], jtype(param[1]))) + let nativeName = emitNative(&"$lib_{toSnakeCase(name)}", params, ret, false) + var publicParams, args: seq[string] + for (pname, t) in params: + let jname = toVarCase(toCamelCase(pname)) + publicParams.add &"{t.java} {jname}" + args.add javaToNative(t, jname) + var ctor = &" public static {name} {constructor.repr}({publicParams.join(\", \")}) {{\n" + ctor.add javaCall(nativeName, args, ret) + ctor.add " }\n\n" + addMember(name, ctor) - cProtos.add &"extern {ret.c} {apiName}({cProtoParams.join(\", \")});\n" +proc emitHandleClass(name, unrefApi: string) = + ## Members shared by ref objects and seqs: the handle, wrapping and + ## releasing. The Cleaner releases on its own thread, and Nim refcounts are + ## not atomic, so objects shared with other threads should be closed + ## explicitly instead of left to the Cleaner. + let unrefNative = emitNative(unrefApi, @[("x", JType(kind: kRef, + native: "long", jni: "jlong", sig: "J", c: "void *"))], + jtype(newEmptyNode()), false) + var code = "" + code.add " final long handle;\n" + code.add " private final java.lang.ref.Cleaner.Cleanable cleanable;\n\n" + code.add &" private {name}(long handle, Void wrapped) {{\n" + code.add " this.handle = handle;\n" + code.add &" this.cleanable = {libClass}.CLEANER.register(this, new Release(handle));\n" + code.add " }\n\n" + code.add &" static {name} wrap(long handle) {{\n" + code.add &" return handle == 0L ? null : new {name}(handle, null);\n" + code.add " }\n\n" + code.add " private static final class Release implements Runnable {\n" + code.add " private final long handle;\n" + code.add " Release(long handle) { this.handle = handle; }\n" + code.add &" @Override public void run() {{ {libClass}.{unrefNative}(handle); }}\n" + code.add " }\n\n" + code.add " /** Releases the Nim object. Do not use this object afterwards. */\n" + code.add " @Override public void close() { cleanable.clean(); }\n\n" + addMember(name, code) - var allJniParams = @["JNIEnv *env", "jclass cls"] - allJniParams.add jniParams - cWrappers.add &"static {ret.jni} {wrapperName}({allJniParams.join(\", \")}) {{\n" - cWrappers.add " (void)cls;\n" - if params.len == 0 or setup.len == 0: - cWrappers.add " (void)env;\n" - cWrappers.add setup.join("") - let call = &"{apiName}({callArgs.join(\", \")})" - case ret.java - of "void": - cWrappers.add &" {call};\n" - cWrappers.add cleanup.join("") - of "String": - # TODO: under ARC/ORC the char* from the internal export points into a - # Nim string that is already freed when it returns. NewStringUTF must copy - # it before anything else allocates; revisit with a threadvar holder. - cWrappers.add &" {ret.c}genny_result = {call};\n" - cWrappers.add cleanup.join("") - cWrappers.add " return (*env)->NewStringUTF(env, genny_result == NULL ? \"\" : genny_result);\n" - of "boolean": - cWrappers.add &" {ret.jni} genny_result = {call} ? JNI_TRUE : JNI_FALSE;\n" - cWrappers.add cleanup.join("") - cWrappers.add " return genny_result;\n" - else: - cWrappers.add &" {ret.jni} genny_result = ({ret.jni}){call};\n" - cWrappers.add cleanup.join("") - cWrappers.add " return genny_result;\n" - cWrappers.add "}\n\n" +proc emitSeqAccessors(className, prefix, methodPrefix: string, selfType, entry: JType) = + ## len/get/set/add/delete/clear for a seq or a seq field. + let + index = jtype(ident"int") + voidT = jtype(newEmptyNode()) + self = ("s", selfType) + proc name(op: string): string = + if methodPrefix == "": op else: methodPrefix & op.capitalizeAscii() + var code = "" + var n = emitNative(prefix & "_len", @[self], index, false) + code.add &" public long {name(\"len\")}() {{\n return {libClass}.{n}(handle);\n }}\n\n" + n = emitNative(prefix & "_get", @[self, ("i", index)], entry, false) + let getCall = nativeToJava(entry, libClass & "." & n & "(handle, i)") + code.add &" public {entry.java} {name(\"get\")}(long i) {{\n" + code.add &" return {getCall};\n }}\n\n" + n = emitNative(prefix & "_set", @[self, ("i", index), ("v", entry)], voidT, false) + code.add &" public void {name(\"set\")}(long i, {entry.java} v) {{\n" + code.add &" {libClass}.{n}(handle, i, {javaToNative(entry, \"v\")});\n }}\n\n" + n = emitNative(prefix & "_add", @[self, ("v", entry)], voidT, false) + code.add &" public void {name(\"add\")}({entry.java} v) {{\n" + code.add &" {libClass}.{n}(handle, {javaToNative(entry, \"v\")});\n }}\n\n" + n = emitNative(prefix & "_delete", @[self, ("i", index)], voidT, false) + code.add &" public void {name(\"delete\")}(long i) {{\n {libClass}.{n}(handle, i);\n }}\n\n" + n = emitNative(prefix & "_clear", @[self], voidT, false) + code.add &" public void {name(\"clear\")}() {{\n {libClass}.{n}(handle);\n }}\n\n" + addMember(className, code) + +proc exportRefObjectJava*( + sym: NimNode, + fields: seq[(string, NimNode)], + constructor: NimNode +) = + let + name = sym.repr + snaked = toSnakeCase(name) + refNames.incl name + classKinds[name] = kRef + let self = jtype(sym) + emitHandleClass(name, &"$lib_{snaked}_unref") + + if constructor != nil: + let ctorType = constructor.getTypeInst() + var params: seq[(string, JType)] + var supported = true + for param in ctorType[0][1 .. ^1]: + let t = jtype(param[1]) + supported = supported and t.kind != kUnsupported + params.add((param[0].repr.split('`')[0], t)) + if supported: + let nativeName = emitNative(&"$lib_{toSnakeCase(constructor.repr)}", + params, self, constructor.raises()) + var publicParams, args: seq[string] + for (pname, t) in params: + let jname = toVarCase(toCamelCase(pname)) + publicParams.add &"{t.java} {jname}" + args.add javaToNative(t, jname) + var code = &" public {name}({publicParams.join(\", \")}) {{\n" + code.add &" this({libClass}.{nativeName}({args.join(\", \")}), null);\n" + code.add " }\n\n" + addMember(name, code) + else: + warning(&"genny java: skipping constructor of {name}, unsupported types", sym) - cMethods.add &" {{\"{sym.repr}\", \"({sig.join(\"\")}){ret.sig}\", (void *){wrapperName}}},\n" + for (fieldName, fieldType) in fields: + let fieldSnaked = toSnakeCase(fieldName) + if fieldType.kind == nnkBracketExpr: + let entry = jtype(fieldType[1]) + if entry.kind == kUnsupported or entry.kind == kString: + warning(&"genny java: skipping field {name}.{fieldName}, unsupported seq entry type", sym) + continue + emitSeqAccessors(name, &"$lib_{snaked}_{fieldSnaked}", fieldName, self, entry) + continue + let t = jtype(fieldType) + if t.kind == kUnsupported: + warning(&"genny java: skipping field {name}.{fieldName}, unsupported type", sym) + continue + let + getter = emitNative(&"$lib_{snaked}_get_{fieldSnaked}", @[("o", self)], t, false) + setter = emitNative(&"$lib_{snaked}_set_{fieldSnaked}", + @[("o", self), ("v", t)], jtype(newEmptyNode()), false) + cap = fieldName.capitalizeAscii() + var code = "" + let getCall = nativeToJava(t, libClass & "." & getter & "(handle)") + code.add &" public {t.java} get{cap}() {{\n" + code.add &" return {getCall};\n }}\n\n" + code.add &" public void set{cap}({t.java} v) {{\n" + code.add &" {libClass}.{setter}(handle, {javaToNative(t, \"v\")});\n }}\n\n" + addMember(name, code) + +proc exportSeqJava*(sym: NimNode) = + let + name = sym.getSeqName() + snaked = toSnakeCase(name) + refNames.incl name + classKinds[name] = kRef + let + self = jtype(sym) + entry = jtype(sym[1]) + emitHandleClass(name, &"$lib_{snaked}_unref") + let newNative = emitNative(&"$lib_new_{snaked}", @[], self, false) + addMember(name, &" public {name}() {{\n this({libClass}.{newNative}(), null);\n }}\n\n") + if entry.kind == kUnsupported: + warning(&"genny java: skipping accessors of {name}, unsupported entry type", sym) + return + emitSeqAccessors(name, &"$lib_{snaked}", "", self, entry) proc writeJava*(dir, lib: string): NimNode = let className = javaClassName(lib) - classPath = gennyJavaPackage.replace('.', '/') & "/" & className javaDir = dir / "java" / gennyJavaPackage.replace('.', '/') cPath = dir / (toSnakeCase(lib) & "_jni.c") javaHome = getEnv("JAVA_HOME") + exceptionClass = className & "Exception" if javaHome == "": error("genny java: JAVA_HOME must be set to a JDK so jni.h can be found") createDir(javaDir) - var java = "" - java.add &"package {gennyJavaPackage};\n\n" + proc header(): string = + &"package {gennyJavaPackage};\n\n" + + var java = header() java.add &"public final class {className} {{\n" java.add &" private {className}() {{}}\n\n" + java.add " static final java.lang.ref.Cleaner CLEANER = java.lang.ref.Cleaner.create();\n\n" java.add " /** Loads the native library from an absolute path. */\n" java.add " public static void load(String absolutePath) {\n" java.add " System.load(absolutePath);\n" java.add " }\n\n" - java.add javaMembers + java.add libMembers java.add "}\n" - writeFile(javaDir / (className & ".java"), java) + writeFile(javaDir / (className & ".java"), java.replace(libClass, className)) + + var exc = header() + exc.add &"public class {exceptionClass} extends RuntimeException {{\n" + exc.add &" public {exceptionClass}(String message) {{ super(message); }}\n" + exc.add "}\n" + writeFile(javaDir / (exceptionClass & ".java"), exc) + + for name, values in enumValues: + var code = header() + code.add &"public enum {name} {{\n" + code.add " " & values.join(", ") & ";\n\n" + code.add &" private static final {name}[] VALUES = values();\n\n" + code.add &" static {name} fromOrdinal(int ordinal) {{ return VALUES[ordinal]; }}\n" + code.add "}\n" + writeFile(javaDir / (name & ".java"), code) + + for name, members in classMembers: + var code = header() + case classKinds.getOrDefault(name, kRef) + of kValue: + code.add &"public final class {name} {{\n" + else: + code.add &"public final class {name} implements AutoCloseable {{\n" + code.add members + code.add "}\n" + writeFile(javaDir / (name & ".java"), code.replace(libClass, className)) var code = "" code.add "#include \n" code.add "#include \n" - code.add "#include \n\n" + code.add "#include \n" + code.add "#include \n\n" code.add "extern void NimMain(void);\n\n" + if needsErrorBridge: + code.add "extern bool $lib_check_error(void);\n" + code.add "extern char *$lib_take_error(void);\n" + code.add "static jclass genny_exception_class;\n" + code.add "static jmethodID genny_enum_ordinal;\n" + for name in enumNames: + code.add &"static jclass genny_class_{name};\n" + code.add &"static jmethodID genny_from_ordinal_{name};\n" + code.add "\n" + code.add "static int genny_load_class(JNIEnv *env, const char *name, jclass *out) {\n" + code.add " jclass local = (*env)->FindClass(env, name);\n" + code.add " if (local == NULL) return 0;\n" + code.add " *out = (jclass)(*env)->NewGlobalRef(env, local);\n" + code.add " (*env)->DeleteLocalRef(env, local);\n" + code.add " return *out != NULL;\n" + code.add "}\n\n" + code.add cTypes code.add cProtos code.add "\n" + code.add cHelpers code.add cWrappers code.add "static JNINativeMethod genny_jni_methods[] = {\n" code.add cMethods @@ -200,10 +652,22 @@ proc writeJava*(dir, lib: string): NimNode = code.add " JNIEnv *env;\n" code.add " if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_8) != JNI_OK) return JNI_ERR;\n" code.add " NimMain();\n" - code.add &" jclass cls = (*env)->FindClass(env, \"{classPath}\");\n" - code.add " if (cls == NULL) return JNI_ERR;\n" + code.add &" if (!genny_load_class(env, \"{classPath(exceptionClass)}\", &genny_exception_class)) return JNI_ERR;\n" + code.add " {\n" + code.add " jclass enumClass = (*env)->FindClass(env, \"java/lang/Enum\");\n" + code.add " if (enumClass == NULL) return JNI_ERR;\n" + code.add " genny_enum_ordinal = (*env)->GetMethodID(env, enumClass, \"ordinal\", \"()I\");\n" + code.add " (*env)->DeleteLocalRef(env, enumClass);\n" + code.add " }\n" + for name in enumNames: + code.add &" if (!genny_load_class(env, \"{classPath(name)}\", &genny_class_{name})) return JNI_ERR;\n" + code.add &" genny_from_ordinal_{name} = (*env)->GetStaticMethodID(env, genny_class_{name}, \"fromOrdinal\", \"(I)L{classPath(name)};\");\n" + code.add &" if (genny_from_ordinal_{name} == NULL) return JNI_ERR;\n" + code.add cOnLoad + code.add " jclass cls;\n" + code.add &" if (!genny_load_class(env, \"{classPath(className)}\", &cls)) return JNI_ERR;\n" code.add " jint count = (jint)(sizeof(genny_jni_methods) / sizeof(genny_jni_methods[0]));\n" - code.add " if (count > 0 && (*env)->RegisterNatives(env, cls, genny_jni_methods, count) != 0) return JNI_ERR;\n" + code.add " if ((*env)->RegisterNatives(env, cls, genny_jni_methods, count) != 0) return JNI_ERR;\n" code.add " return JNI_VERSION_1_8;\n" code.add "}\n" code = code.replace("$lib", toSnakeCase(lib)) diff --git a/tests/TestJavaMain.java b/tests/TestJavaMain.java index f0051e2..e37d356 100644 --- a/tests/TestJavaMain.java +++ b/tests/TestJavaMain.java @@ -1,4 +1,4 @@ -import genny.TestJava; +import genny.*; public class TestJavaMain { static void check(boolean ok, String what) { @@ -27,6 +27,40 @@ public static void main(String[] args) { for (int i = 0; i < 3; i++) TestJava.bump(); check(TestJava.getCallCount() == 3, "void proc + state"); + check(TestJava.nextColor(Color.BLUE) == Color.RED, "enum param and return"); + + Tile tile = TestJava.moveTile(new Tile(new Vec(1f, 2f), Color.GREEN, true), 0.5f, -1f); + check(tile.pos.x == 1.5f && tile.pos.y == 1f, "nested value object"); + check(tile.color == Color.GREEN && tile.solid, "value object enum and bool fields"); + + try (Counter counter = new Counter(10)) { + counter.bumpCounter(); + counter.bumpCounter(); + check(counter.getCount() == 12, "ref object constructor and proc"); + counter.setColor(Color.BLUE); + check(counter.getColor() == Color.BLUE, "ref object enum field"); + check(counter.historyLen() == 2 && counter.historyGet(1) == 12, "ref object seq field"); + } + + try (SeqString w = TestJava.words()) { + check(w.len() == 2 && w.get(1).equals("beta"), "seq[string]"); + w.add("gamma"); + check(w.get(2).equals("gamma"), "seq[string] add"); + } + + check(TestJava.failIfNegative(5) == 5, "raising proc, no error"); + try { + TestJava.failIfNegative(-3); + check(false, "raising proc throws"); + } catch (TestJavaException e) { + check(e.getMessage().equals("negative: -3"), "raising proc throws"); + } + + for (int i = 0; i < 10000; i++) new Counter(i); + System.gc(); + try { Thread.sleep(200); } catch (InterruptedException e) {} + check(true, "Cleaner releases unclosed ref objects"); + System.out.println("All Java tests passed"); } } diff --git a/tests/TestJavaStandard.java b/tests/TestJavaStandard.java new file mode 100644 index 0000000..d7d30a3 --- /dev/null +++ b/tests/TestJavaStandard.java @@ -0,0 +1,59 @@ +import genny.*; + +public class TestJavaStandard { + static void check(boolean ok, String what) { + if (!ok) throw new AssertionError(what); + System.out.println("ok " + what); + } + + public static void main(String[] args) { + Test.load(new java.io.File(args[0]).getAbsolutePath()); + + check(Test.simpleCall(42) == 42, "simpleCall"); + check(Test.simpleCall(0) == 0, "simpleCall zero"); + check(Test.SIMPLE_CONST == 123L, "SIMPLE_CONST"); + check(SimpleEnum.values().length == 3 && SimpleEnum.THIRD.ordinal() == 2, "SimpleEnum"); + + SimpleObj obj = new SimpleObj(10, (byte)20, true); + check(obj.simpleA == 10 && obj.simpleB == 20 && obj.simpleC, "SimpleObj fields"); + check(obj.equals(new SimpleObj(10, (byte)20, true)), "SimpleObj equals"); + + try (SimpleRefObj ref = new SimpleRefObj()) { + ref.setSimpleRefA(100); + check(ref.getSimpleRefA() == 100, "SimpleRefObj a"); + ref.setSimpleRefB((byte)50); + check(ref.getSimpleRefB() == 50, "SimpleRefObj b"); + ref.doit(); + } + + try (SeqInt seq = new SeqInt()) { + seq.add(1); + seq.add(2); + seq.add(3); + check(seq.len() == 3, "SeqInt len"); + check(seq.get(0) == 1 && seq.get(1) == 2 && seq.get(2) == 3, "SeqInt get"); + seq.set(1, 20); + check(seq.get(1) == 20, "SeqInt set"); + seq.delete(0); + check(seq.len() == 2, "SeqInt delete"); + seq.clear(); + check(seq.len() == 0, "SeqInt clear"); + } + + try (RefObjWithSeq r = new RefObjWithSeq()) { + r.dataAdd((byte)7); + r.dataAdd((byte)8); + check(r.dataLen() == 2 && r.dataGet(1) == 8, "RefObjWithSeq data"); + } + + new SimpleObjWithProc(1, (byte)2, false).extraProc(); + check(true, "SimpleObjWithProc.extraProc"); + + try (SeqString datas = Test.getDatas()) { + check(datas.len() == 3, "getDatas len"); + check(datas.get(0).equals("a") && datas.get(2).equals("c"), "getDatas get"); + } + + System.out.println("All Java standard tests passed"); + } +} diff --git a/tests/test_java.nim b/tests/test_java.nim index 6040dc4..bc64454 100644 --- a/tests/test_java.nim +++ b/tests/test_java.nim @@ -54,6 +54,89 @@ exportProcs: bump getCallCount -writeFiles("tests/generated_java", "testJava") +type + Color = enum + Red + Green + Blue + + Vec = object + x*, y*: float32 + + Tile = object + pos*: Vec + color*: Color + solid*: bool + + Counter* = ref object + count*: int + color*: Color + history*: seq[int] + + TestJavaError = object of ValueError + +var lastError: ref TestJavaError + +exportEnums: + Color + +exportObject Vec: + discard + +exportObject Tile: + discard + +proc nextColor(c: Color): Color = + Color((c.ord + 1) mod 3) + +proc moveTile(t: Tile, dx, dy: float32): Tile = + result = t + result.pos.x += dx + result.pos.y += dy + +proc newCounter(start: int): Counter = + Counter(count: start) + +proc bumpCounter(c: Counter) = + inc c.count + c.history.add c.count + +exportRefObject Counter: + fields: + count + color + history + constructor: + newCounter(int) + procs: + bumpCounter + +proc words(): seq[string] = + @["alpha", "beta"] + +exportSeq seq[string]: + discard + +proc checkError(): bool = + lastError != nil + +proc takeError(): string = + result = lastError.msg + lastError = nil + +proc failIfNegative(x: int): int {.raises: [TestJavaError].} = + if x < 0: + raise newException(TestJavaError, "negative: " & $x) + x + +exportProcs: + nextColor + moveTile + words + checkError + takeError + failIfNegative + +writeFiles("tests/generated_java", "TestJava") include generated_java/internal From 573cd54eeedfd140a6141a41d49b012e12a32c76 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 22:27:33 -0400 Subject: [PATCH 05/10] Java backend: pass Nim pointer params as direct ByteBuffers A pointer parameter maps to java.nio.ByteBuffer and the wrapper passes GetDirectBufferAddress, so Java and Nim can share memory without copies. Pointer returns stay unsupported. Co-Authored-By: Claude Opus 5.5 --- src/genny/languages/java.nim | 9 +++++++-- tests/TestJavaMain.java | 4 ++++ tests/test_java.nim | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index 1512c19..687a99d 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -11,7 +11,7 @@ when not defined(noSignalHandler): type Kind = enum - kVoid, kPrim, kString, kEnum, kValue, kRef, kUnsupported + kVoid, kPrim, kString, kEnum, kValue, kRef, kBuffer, kUnsupported JType = object kind: Kind @@ -95,6 +95,10 @@ proc jtype(sym: NimNode): JType = prim("float", "jfloat", "F", "float", "Float") of "float", "float64": prim("double", "jdouble", "D", "double", "Double") + of "pointer": + JType(kind: kBuffer, java: "java.nio.ByteBuffer", native: "java.nio.ByteBuffer", + jni: "jobject", sig: "Ljava/nio/ByteBuffer;", + publicSig: "Ljava/nio/ByteBuffer;", c: "void *") of "string": JType(kind: kString, java: "String", native: "String", jni: "jstring", sig: "Ljava/lang/String;", publicSig: "Ljava/lang/String;", c: "char *") @@ -133,6 +137,7 @@ proc cArg(t: JType, name: string, setup, cleanup: var string): string = of kEnum: &"({t.c}){name}" of kValue: &"genny_to_{t.c}(env, {name})" of kRef: &"(void *)(intptr_t){name}" + of kBuffer: &"({name} == NULL ? NULL : (*env)->GetDirectBufferAddress(env, {name}))" else: name proc cReturn(t: JType, expr: string): string = @@ -263,7 +268,7 @@ proc exportProcJava*( apiName.add toSnakeCase(sym.repr) var - supported = ret.kind != kUnsupported + supported = ret.kind notin {kUnsupported, kBuffer} params: seq[(string, JType)] for param in procParams: for i in 0 .. param.len - 3: diff --git a/tests/TestJavaMain.java b/tests/TestJavaMain.java index e37d356..23a49fd 100644 --- a/tests/TestJavaMain.java +++ b/tests/TestJavaMain.java @@ -56,6 +56,10 @@ public static void main(String[] args) { check(e.getMessage().equals("negative: -3"), "raising proc throws"); } + java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocateDirect(16).order(java.nio.ByteOrder.nativeOrder()); + TestJava.fillBuffer(buf, 16, 40); + check(buf.getInt(0) == 40 && buf.getInt(12) == 43, "direct ByteBuffer as pointer"); + for (int i = 0; i < 10000; i++) new Counter(i); System.gc(); try { Thread.sleep(200); } catch (InterruptedException e) {} diff --git a/tests/test_java.nim b/tests/test_java.nim index bc64454..cb20295 100644 --- a/tests/test_java.nim +++ b/tests/test_java.nim @@ -117,6 +117,11 @@ proc words(): seq[string] = exportSeq seq[string]: discard +proc fillBuffer(buf: pointer, len: int, value: int32) = + let bytes = cast[ptr UncheckedArray[int32]](buf) + for i in 0 ..< len div 4: + bytes[i] = value + i.int32 + proc checkError(): bool = lastError != nil @@ -130,6 +135,7 @@ proc failIfNegative(x: int): int {.raises: [TestJavaError].} = x exportProcs: + fillBuffer nextColor moveTile words From ce7da2ba406c5f2741238738aacec0aa4f42d3c8 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 23:10:26 -0400 Subject: [PATCH 06/10] Java backend: harden the JNI boundary - Only call NimMain from JNI_OnLoad when Nim does not initialize itself (--app:lib without --noMain already runs it from a load constructor), so top-level code runs once. - Every JNI conversion can fail and jumps to cleanup before calling Nim: heap ByteBuffers throw IllegalArgumentException, null value objects and enums throw NullPointerException, and Call*/NewObject results are exception-checked. -Xcheck:jni now reports no warnings. - Strings are transcoded between UTF-16 and standard UTF-8 instead of passing JNI modified UTF-8; invalid UTF-8 from Nim becomes U+FFFD and exception messages go through the same path. - Pointer fields and seq entries are skipped; emitNative refuses pointer returns. - Cleaner releases are queued and run on the next call into the library (or releasePending()), so Nim refcounts are only touched by calling threads. Handle-bearing calls keep their Java objects reachable with reachabilityFence, and closed objects throw IllegalStateException. - Enums use the real C size and explicit ordinal values; Java enums carry a value field instead of relying on ordinal(). - Value objects that Nim would not pass as plain C structs (over 24 bytes without {.bycopy.}, inherited, packed, union, variant) are skipped. - Constants are rendered as Java literals from their values. - Overloads with the same Java signature get the overload prefix appended; other method clashes and types named like the generated classes fail at compile time. Java keywords get a trailing underscore. - Seq get/set/delete check bounds and throw IndexOutOfBoundsException. - Value object equals/hashCode use Float/Double.compare semantics. - Platform JNI include directory, quoted include paths, JNICALL wrappers, pointer-sized C type for Nim int, java.lang names fully qualified. Co-Authored-By: Claude Opus 5.5 --- src/genny/languages/java.nim | 911 ++++++++++++++++++++++++++--------- tests/TestJavaMain.java | 99 +++- tests/test_java.nim | 102 +++- 3 files changed, 864 insertions(+), 248 deletions(-) diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index 687a99d..3375d73 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -1,5 +1,5 @@ import - std/[macros, os, sets, strformat, strutils, tables], + std/[macros, os, sets, strformat, strutils, tables, unicode], ../common const gennyJavaPackage {.strdefine.} = "genny" @@ -23,18 +23,31 @@ type c: string ## C type of the Nim export. field: string ## GetField / New suffix for primitives. -const libClass = "$LibClass" +const + libClass = "$LibClass" + # Nim passes object parameters larger than this by pointer unless they + # are {.bycopy.}; the JNI wrappers pass value objects as C structs. + maxByValueSize = 3 * 8 + javaKeywords = [ + "abstract", "assert", "boolean", "break", "byte", "case", "catch", + "char", "class", "const", "continue", "default", "do", "double", "else", + "enum", "extends", "false", "final", "finally", "float", "for", "goto", + "if", "implements", "import", "instanceof", "int", "interface", "long", + "native", "new", "null", "package", "private", "protected", "public", + "return", "short", "static", "strictfp", "super", "switch", + "synchronized", "this", "throw", "throws", "transient", "true", "try", + "void", "volatile", "while", "yield", "_" + ] var libMembers {.compiletime.}: string classMembers {.compiletime.}: OrderedTable[string, string] classKinds {.compiletime.}: Table[string, Kind] - enumNames {.compiletime.}: HashSet[string] - enumSizes {.compiletime.}: Table[string, int] - enumValues {.compiletime.}: Table[string, seq[string]] + enumCTypes {.compiletime.}: Table[string, string] + enumEntries {.compiletime.}: OrderedTable[string, seq[(string, int)]] valueNames {.compiletime.}: HashSet[string] - valueFields {.compiletime.}: Table[string, seq[(string, NimNode)]] refNames {.compiletime.}: HashSet[string] + javaSigs {.compiletime.}: HashSet[string] cTypes {.compiletime.}: string cProtos {.compiletime.}: string cHelpers {.compiletime.}: string @@ -42,6 +55,7 @@ var cMethods {.compiletime.}: string cOnLoad {.compiletime.}: string needsErrorBridge {.compiletime.}: bool + needsStrings {.compiletime.}: bool proc stripSink(sym: NimNode): NimNode = if sym.kind == nnkBracketExpr and sym[0].repr == "sink": @@ -55,6 +69,29 @@ proc isVoid(sym: NimNode): bool = proc classPath(name: string): string = gennyJavaPackage.replace('.', '/') & "/" & name +proc javaIdent(name: string): string = + if name in javaKeywords: name & "_" else: name + +proc javaName(snaked: string): string = + ## Java identifier for a snake_case Nim name. + javaIdent(toVarCase(toCamelCase(snaked))) + +proc javaMethodName(sym: NimNode): string = + ## Java method name for a Nim proc, or "" when there is none. + let name = sym.repr.strip(chars = {'`'}) + if name.len == 0 or name[0] notin IdentStartChars: + return "" + for c in name: + if c notin IdentChars: + return "" + javaIdent(name) + +proc claim(className, name: string, params: seq[string]): bool = + ## Reserves a Java method signature, false when it is already taken. + let key = &"{className}.{name}({params.join(\",\")})" + result = key notin javaSigs + javaSigs.incl key + proc prim(java, jni, sig, c, field: string): JType = JType(kind: kPrim, java: java, native: java, jni: jni, sig: sig, publicSig: sig, c: c, field: field) @@ -70,10 +107,9 @@ proc jtype(sym: NimNode): JType = return JType(kind: kRef, java: name, native: "long", jni: "jlong", sig: "J", publicSig: &"L{classPath(name)};", c: "void *") return JType(kind: kUnsupported) - if name in enumNames: - let c = if enumSizes[name] <= 1: "uint8_t" else: "uint16_t" + if name in enumCTypes: return JType(kind: kEnum, java: name, native: "int", jni: "jint", - sig: "I", publicSig: &"L{classPath(name)};", c: c) + sig: "I", publicSig: &"L{classPath(name)};", c: enumCTypes[name]) if name in valueNames: return JType(kind: kValue, java: name, native: name, jni: "jobject", sig: &"L{classPath(name)};", publicSig: &"L{classPath(name)};", c: name) @@ -81,78 +117,44 @@ proc jtype(sym: NimNode): JType = return JType(kind: kRef, java: name, native: "long", jni: "jlong", sig: "J", publicSig: &"L{classPath(name)};", c: "void *") case name - of "int", "int64", "uint", "uint64": - prim("long", "jlong", "J", "int64_t", "Long") - of "int32", "uint32": - prim("int", "jint", "I", "int32_t", "Int") - of "int16", "uint16": - prim("short", "jshort", "S", "int16_t", "Short") - of "int8", "uint8", "byte": - prim("byte", "jbyte", "B", "int8_t", "Byte") - of "bool": - prim("boolean", "jboolean", "Z", "bool", "Boolean") - of "float32": - prim("float", "jfloat", "F", "float", "Float") - of "float", "float64": - prim("double", "jdouble", "D", "double", "Double") + of "int": prim("long", "jlong", "J", "intptr_t", "Long") + of "uint": prim("long", "jlong", "J", "uintptr_t", "Long") + of "int64": prim("long", "jlong", "J", "int64_t", "Long") + of "uint64": prim("long", "jlong", "J", "uint64_t", "Long") + of "int32": prim("int", "jint", "I", "int32_t", "Int") + of "uint32": prim("int", "jint", "I", "uint32_t", "Int") + of "int16": prim("short", "jshort", "S", "int16_t", "Short") + of "uint16": prim("short", "jshort", "S", "uint16_t", "Short") + of "int8": prim("byte", "jbyte", "B", "int8_t", "Byte") + of "uint8", "byte": prim("byte", "jbyte", "B", "uint8_t", "Byte") + of "bool": prim("boolean", "jboolean", "Z", "bool", "Boolean") + of "float32": prim("float", "jfloat", "F", "float", "Float") + of "float", "float64": prim("double", "jdouble", "D", "double", "Double") of "pointer": JType(kind: kBuffer, java: "java.nio.ByteBuffer", native: "java.nio.ByteBuffer", jni: "jobject", sig: "Ljava/nio/ByteBuffer;", publicSig: "Ljava/nio/ByteBuffer;", c: "void *") of "string": - JType(kind: kString, java: "String", native: "String", jni: "jstring", - sig: "Ljava/lang/String;", publicSig: "Ljava/lang/String;", c: "char *") + JType(kind: kString, java: "java.lang.String", native: "java.lang.String", + jni: "jstring", sig: "Ljava/lang/String;", + publicSig: "Ljava/lang/String;", c: "char *") else: JType(kind: kUnsupported) proc javaToNative(t: JType, expr: string): string = ## Converts a public Java value to the value passed to `native`. case t.kind - of kEnum: &"{expr}.ordinal()" - of kRef: &"({expr} == null ? 0L : {expr}.handle)" + of kEnum: &"{expr}.value" + of kRef: + if expr == "this": "this.handle$()" + else: &"({expr} == null ? 0L : {expr}.handle$())" else: expr proc nativeToJava(t: JType, expr: string): string = ## Converts a `native` return value to the public Java value. case t.kind - of kEnum: &"{t.java}.fromOrdinal({expr})" - of kRef: &"{t.java}.wrap({expr})" - else: expr - -proc zeroReturn(t: JType): string = - case t.kind - of kVoid: "return;" - of kString, kValue: "return NULL;" - else: "return 0;" - -proc cArg(t: JType, name: string, setup, cleanup: var string): string = - ## Converts a JNI argument to the C value the Nim export takes. - case t.kind - of kString: - setup.add &" const char *{name}_c = {name} == NULL ? \"\" : (*env)->GetStringUTFChars(env, {name}, NULL);\n" - cleanup.add &" if ({name} != NULL) (*env)->ReleaseStringUTFChars(env, {name}, {name}_c);\n" - &"(char *){name}_c" - of kPrim: - if t.java == "boolean": &"{name} == JNI_TRUE" else: &"({t.c}){name}" - of kEnum: &"({t.c}){name}" - of kValue: &"genny_to_{t.c}(env, {name})" - of kRef: &"(void *)(intptr_t){name}" - of kBuffer: &"({name} == NULL ? NULL : (*env)->GetDirectBufferAddress(env, {name}))" - else: name - -proc cReturn(t: JType, expr: string): string = - ## Converts the C value a Nim export returned to a JNI value. - case t.kind - of kString: - # TODO: under ARC/ORC the char* from the internal export points into a - # Nim string that is already freed when it returns. NewStringUTF must - # copy it before anything else allocates; revisit with a threadvar holder. - &"(*env)->NewStringUTF(env, {expr} == NULL ? \"\" : {expr})" - of kPrim: - if t.java == "boolean": &"({expr} ? JNI_TRUE : JNI_FALSE)" else: &"({t.jni}){expr}" - of kEnum: &"(jint){expr}" - of kValue: &"genny_from_{t.c}(env, {expr})" - of kRef: &"(jlong)(intptr_t){expr}" + of kEnum: &"{t.java}.fromValue({expr})" + of kRef: &"{t.java}.wrap$({expr})" else: expr proc javaClassName(lib: string): string = @@ -172,54 +174,187 @@ proc emitNative( ): string = ## Emits the JNI wrapper for one Nim export and returns the Java native ## method name, declared on the library class. + doAssert ret.kind notin {kBuffer, kUnsupported} let nativeName = "n_" & apiName.replace("$lib_", "") wrapperName = "genny_jni_" & apiName.replace("$lib_", "") var javaParams, cProtoParams, jniParams, sig, callArgs: seq[string] - var setup, cleanup: string + var decls, setup, cleanup: string for i, (name, t) in params: - let pname = &"p{i}" - javaParams.add &"{t.native} {pname}" - cProtoParams.add &"{t.c} {name}" - jniParams.add &"{t.jni} {pname}" + let p = &"p{i}" + javaParams.add &"{t.native} {p}" + cProtoParams.add t.c + jniParams.add &"{t.jni} {p}" sig.add t.sig - callArgs.add cArg(t, pname, setup, cleanup) + case t.kind + of kString: + needsStrings = true + decls.add &" char *{p}_c = NULL;\n" + setup.add &" {p}_c = genny_utf8_from_jstring(env, {p});\n" + setup.add &" if ({p}_c == NULL) goto genny_done;\n" + cleanup.add &" free({p}_c);\n" + callArgs.add &"{p}_c" + of kValue: + decls.add &" {t.c} {p}_c;\n" + setup.add &" if (!genny_to_{t.c}(env, {p}, &{p}_c)) goto genny_done;\n" + callArgs.add &"{p}_c" + of kBuffer: + decls.add &" void *{p}_c = NULL;\n" + setup.add &" if ({p} != NULL) {{\n" + setup.add &" {p}_c = (*env)->GetDirectBufferAddress(env, {p});\n" + setup.add &" if ({p}_c == NULL) {{\n" + setup.add " genny_throw(env, \"java/lang/IllegalArgumentException\", \"pointer arguments must be direct ByteBuffers\");\n" + setup.add " goto genny_done;\n" + setup.add " }\n" + setup.add " }\n" + callArgs.add &"{p}_c" + of kPrim: + callArgs.add(if t.java == "boolean": &"{p} == JNI_TRUE" else: &"({t.c}){p}") + of kEnum: + callArgs.add &"({t.c}){p}" + of kRef: + callArgs.add &"(void *)(intptr_t){p}" + else: + callArgs.add p libMembers.add &" static native {ret.native} {nativeName}({javaParams.join(\", \")});\n" cProtos.add &"extern {ret.c} {apiName}({cProtoParams.join(\", \")});\n" var allJniParams = @["JNIEnv *env", "jclass cls"] allJniParams.add jniParams - cWrappers.add &"static {ret.jni} {wrapperName}({allJniParams.join(\", \")}) {{\n" + cWrappers.add &"static {ret.jni} JNICALL {wrapperName}({allJniParams.join(\", \")}) {{\n" + case ret.kind + of kVoid: discard + of kString, kValue: cWrappers.add &" {ret.jni} genny_ret = NULL;\n" + else: cWrappers.add &" {ret.jni} genny_ret = 0;\n" + cWrappers.add decls cWrappers.add " (void)env; (void)cls;\n" cWrappers.add setup let call = &"{apiName}({callArgs.join(\", \")})" + cWrappers.add " {\n" if ret.kind == kVoid: - cWrappers.add &" {call};\n" + cWrappers.add &" {call};\n" else: - cWrappers.add &" {ret.c} genny_result = {call};\n" - cWrappers.add cleanup + cWrappers.add &" {ret.c} genny_result = {call};\n" + var store = "" + case ret.kind + of kVoid: discard + of kString: + # TODO: under ARC/ORC the char* from the internal export points into a + # Nim string that is already freed when it returns. It is copied here + # before anything else allocates; revisit with a threadvar holder. + needsStrings = true + store = "genny_ret = genny_jstring_from_utf8(env, genny_result);" + of kPrim: + store = + if ret.java == "boolean": "genny_ret = genny_result ? JNI_TRUE : JNI_FALSE;" + else: &"genny_ret = ({ret.jni})genny_result;" + of kEnum: store = "genny_ret = (jint)genny_result;" + of kValue: store = &"genny_ret = genny_from_{ret.c}(env, genny_result);" + of kRef: store = "genny_ret = (jlong)(intptr_t)genny_result;" + else: discard if raises: needsErrorBridge = true - cWrappers.add " if ($lib_check_error()) {\n" - cWrappers.add " char *genny_error = $lib_take_error();\n" - cWrappers.add " (*env)->ThrowNew(env, genny_exception_class, genny_error == NULL ? \"error\" : genny_error);\n" - cWrappers.add &" {zeroReturn(ret)}\n" - cWrappers.add " }\n" + needsStrings = true + cWrappers.add " if ($lib_check_error()) genny_raise(env);\n" + if store != "": + cWrappers.add &" else {store}\n" + elif store != "": + cWrappers.add &" {store}\n" + cWrappers.add " }\n" + if "genny_done" in setup: + cWrappers.add "genny_done:;\n" + cWrappers.add cleanup if ret.kind != kVoid: - cWrappers.add &" return {cReturn(ret, \"genny_result\")};\n" + cWrappers.add " return genny_ret;\n" cWrappers.add "}\n\n" cMethods.add &" {{\"{nativeName}\", \"({sig.join(\"\")}){ret.sig}\", (void *){wrapperName}}},\n" nativeName -proc javaCall(nativeName: string, args: seq[string], ret: JType): string = +proc javaMethod( + signature, nativeName: string, + args: seq[string], + ret: JType, + fences: seq[string], + checks: seq[string] = @[] +): string = + ## A public Java method that calls one native method. Pending releases + ## run first, and ref objects stay reachable until the call returns. let call = &"{libClass}.{nativeName}({args.join(\", \")})" + var lines = checks if ret.kind == kVoid: - &" {call};\n" + lines.add call & ";" + else: + lines.add &"return {nativeToJava(ret, call)};" + result = &" {signature} {{\n" + result.add &" {libClass}.releasePending();\n" + if fences.len == 0: + for line in lines: + result.add &" {line}\n" else: - &" return {nativeToJava(ret, call)};\n" + result.add " try {\n" + for line in lines: + result.add &" {line}\n" + result.add " } finally {\n" + for fence in fences: + result.add &" java.lang.ref.Reference.reachabilityFence({fence});\n" + result.add " }\n" + result.add " }\n\n" + +proc javaString(s: string): string = + ## Java string literal; control characters use octal escapes because + ## javac translates \u escapes before it reads string literals. + result = "\"" + for rune in s.runes: + let c = rune.int + if c == '"'.ord: + result.add "\\\"" + elif c == '\\'.ord: + result.add "\\\\" + elif c < 0x20 or c == 0x7F: + result.add "\\" & toOct(c, 3) + elif c < 0x7F: + result.add chr(c) + elif c > 0xFFFF: + let v = c - 0x10000 + result.add "\\u" & toHex(0xD800 + (v shr 10), 4) + result.add "\\u" & toHex(0xDC00 + (v and 0x3FF), 4) + else: + result.add "\\u" & toHex(c, 4) + result.add "\"" + +proc wrapSigned(v: BiggestInt, bits: int): BiggestInt = + ## Reinterprets the low `bits` bits of v as a signed integer. + let span = 1'i64 shl bits + result = v and (span - 1) + if result >= span div 2: + result -= span + +proc javaLiteral(t: JType, v: NimNode): string = + ## Java literal for a constant value node. + case t.java + of "java.lang.String": javaString(v.strVal) + of "boolean": (if v.intVal != 0: "true" else: "false") + of "long": $v.intVal & "L" + of "int": $wrapSigned(v.intVal, 32) + of "short": &"(short)({wrapSigned(v.intVal, 16)})" + of "byte": &"(byte)({wrapSigned(v.intVal, 8)})" + else: + let + f = v.floatVal + box = if t.java == "float": "java.lang.Float" else: "java.lang.Double" + if f != f: + &"{box}.NaN" + elif f == Inf: + &"{box}.POSITIVE_INFINITY" + elif f == NegInf: + &"{box}.NEGATIVE_INFINITY" + elif t.java == "float": + $f & "f" + else: + $f proc exportConstJava*(sym: NimNode) = let t = jtype(sym.getTypeInst()) @@ -229,24 +364,41 @@ proc exportConstJava*(sym: NimNode) = var valueNode = sym.getImpl()[2] while valueNode.kind == nnkSym: valueNode = valueNode.getImpl()[2] - var value = valueNode.repr - case t.java - of "long": value.add "L" - of "float": value.add "f" - of "short", "byte": value = &"({t.java}){value}" - else: discard - libMembers.add &" public static final {t.java} {toCapSnakeCase(sym.repr)} = {value};\n" + libMembers.add &" public static final {t.java} {toCapSnakeCase(sym.repr)} = {javaLiteral(t, valueNode)};\n" proc exportEnumJava*(sym: NimNode) = - let - name = sym.repr - entries = sym.getImpl()[2][1 .. ^1] - var values: seq[string] - for entry in entries: - values.add toCapSnakeCase(entry.repr) - enumNames.incl name - enumSizes[name] = if entries.len <= 256: 1 else: 2 - enumValues[name] = values + let name = sym.repr + var + entries: seq[(string, int)] + next = 0 + for entry in sym.getImpl()[2][1 .. ^1]: + var + field = entry + value = next + if entry.kind == nnkEnumFieldDef: + field = entry[0] + case entry[1].kind + of nnkIntLit .. nnkUInt64Lit: + value = entry[1].intVal.int + of nnkTupleConstr, nnkPar: + value = entry[1][0].intVal.int + else: + discard + if value < int32.low or value > int32.high: + warning(&"genny java: skipping enum {name}, values must fit in a Java int", sym) + return + entries.add((toCapSnakeCase(field.repr), value)) + next = value + 1 + # Matches the C type Nim's code generator picks for the enum. + enumCTypes[name] = + if entries[0][1] < 0: "int32_t" + else: + case getSize(sym) + of 1: "uint8_t" + of 2: "uint16_t" + of 8: "int64_t" + else: "int32_t" + enumEntries[name] = entries classKinds[name] = kEnum proc exportProcJava*( @@ -278,32 +430,70 @@ proc exportProcJava*( if not supported: warning(&"genny java: skipping proc {sym.repr}, unsupported types", sym) return - - let nativeName = emitNative(apiName, params, ret, sym.raises()) + var methodName = javaMethodName(sym) + if methodName == "": + warning(&"genny java: skipping proc {sym.repr}, no Java name for it", sym) + return let onType = owner != nil and params.len > 0 - var publicParams, args: seq[string] + var publicParams, publicTypes, args, fences: seq[string] for i, (name, t) in params: - if i == 0 and onType: - args.add javaToNative(t, "this") - else: - let jname = toVarCase(toCamelCase(name)) + let jname = if i == 0 and onType: "this" else: javaName(name) + if jname != "this": publicParams.add &"{t.java} {jname}" - args.add javaToNative(t, jname) + publicTypes.add t.java + args.add javaToNative(t, jname) + if t.kind == kRef: + fences.add jname let className = if onType: owner.getName() else: libClass modifier = if onType: "public" else: "public static" - var code = &" {modifier} {ret.java} {sym.repr}({publicParams.join(\", \")}) {{\n" - code.add javaCall(nativeName, args, ret) - code.add " }\n" - addMember(className, code) + if not claim(className, methodName, publicTypes): + var renamed = methodName + for prefix in prefixes: + renamed.add toCamelCase(toSnakeCase(prefix.getName())) + if renamed == methodName or not claim(className, renamed, publicTypes): + error(&"genny java: {sym.repr} has the same Java signature as another exported method", sym) + methodName = renamed + + let nativeName = emitNative(apiName, params, ret, sym.raises()) + addMember(className, javaMethod( + &"{modifier} {ret.java} {methodName}({publicParams.join(\", \")})", + nativeName, args, ret, fences)) + +proc fieldIdent(property: NimNode): NimNode = + result = property + while result.kind in {nnkPostfix, nnkPragmaExpr, nnkAccQuoted}: + result = if result.kind == nnkPostfix: result[1] else: result[0] + +proc valueLayoutProblem(sym: NimNode): string = + ## Why the object cannot cross JNI as a plain C struct, or "". + let impl = sym.getImpl() + var pragmas: seq[string] + if impl[0].kind == nnkPragmaExpr: + for pragma in impl[0][1]: + pragmas.add(if pragma.kind in {nnkExprColonExpr, nnkCall}: pragma[0].repr else: pragma.repr) + let obj = impl[2] + if obj.kind != nnkObjectTy: + return "it is not a plain object" + if obj[1].kind != nnkEmpty: + return "inherited objects are not supported" + for pragma in ["packed", "union", "byref", "inheritable"]: + if pragma in pragmas: + return &"{{.{pragma}.}} objects are not supported" + if obj[2].kind == nnkRecList: + for item in obj[2]: + if item.kind != nnkIdentDefs: + return "object variants are not supported" + if "bycopy" notin pragmas and getSize(sym) > maxByValueSize: + return &"Nim passes objects over {maxByValueSize} bytes by pointer, mark it {{.bycopy.}}" proc emitValueConversions(name: string, fields: seq[(string, NimNode)]) = ## C helpers between the Java value class and the Nim struct. cTypes.add &"typedef struct {name} {{\n" for (fieldName, fieldType) in fields: - cTypes.add &" {jtype(fieldType).c} {fieldName};\n" + cTypes.add &" {jtype(fieldType).c} f_{fieldName};\n" cTypes.add &"}} {name};\n\n" cHelpers.add &"static jclass genny_class_{name};\n" @@ -311,10 +501,9 @@ proc emitValueConversions(name: string, fields: seq[(string, NimNode)]) = for (fieldName, _) in fields: cHelpers.add &"static jfieldID genny_field_{name}_{fieldName};\n" - cHelpers.add &"static {name} genny_to_{name}(JNIEnv *env, jobject o) {{\n" - cHelpers.add &" {name} r;\n" - cHelpers.add " memset(&r, 0, sizeof(r));\n" - cHelpers.add " if (o == NULL) return r;\n" + cHelpers.add &"static int genny_to_{name}(JNIEnv *env, jobject o, {name} *r) {{\n" + cHelpers.add " memset(r, 0, sizeof(*r));\n" + cHelpers.add &" if (o == NULL) {{ genny_throw(env, \"java/lang/NullPointerException\", \"{name} is null\"); return 0; }}\n" for (fieldName, fieldType) in fields: let t = jtype(fieldType) @@ -322,42 +511,53 @@ proc emitValueConversions(name: string, fields: seq[(string, NimNode)]) = case t.kind of kPrim: if t.java == "boolean": - cHelpers.add &" r.{fieldName} = (*env)->GetBooleanField(env, o, {fid}) == JNI_TRUE;\n" + cHelpers.add &" r->f_{fieldName} = (*env)->GetBooleanField(env, o, {fid}) == JNI_TRUE;\n" else: - cHelpers.add &" r.{fieldName} = ({t.c})(*env)->Get{t.field}Field(env, o, {fid});\n" + cHelpers.add &" r->f_{fieldName} = ({t.c})(*env)->Get{t.field}Field(env, o, {fid});\n" of kEnum: cHelpers.add &" {{ jobject e = (*env)->GetObjectField(env, o, {fid});\n" - cHelpers.add &" r.{fieldName} = e == NULL ? 0 : ({t.c})(*env)->CallIntMethod(env, e, genny_enum_ordinal);\n" - cHelpers.add " if (e != NULL) (*env)->DeleteLocalRef(env, e); }\n" + cHelpers.add &" if (e == NULL) {{ genny_throw(env, \"java/lang/NullPointerException\", \"{name}.{javaName(fieldName)} is null\"); return 0; }}\n" + cHelpers.add &" r->f_{fieldName} = ({t.c})(*env)->GetIntField(env, e, genny_enum_value_{t.java});\n" + cHelpers.add " (*env)->DeleteLocalRef(env, e); }\n" of kValue: cHelpers.add &" {{ jobject v = (*env)->GetObjectField(env, o, {fid});\n" - cHelpers.add &" r.{fieldName} = genny_to_{t.c}(env, v);\n" - cHelpers.add " if (v != NULL) (*env)->DeleteLocalRef(env, v); }\n" + cHelpers.add &" int ok = genny_to_{t.c}(env, v, &r->f_{fieldName});\n" + cHelpers.add " if (v != NULL) (*env)->DeleteLocalRef(env, v);\n" + cHelpers.add " if (!ok) return 0; }\n" else: discard - cHelpers.add " return r;\n}\n\n" + cHelpers.add " return 1;\n}\n\n" - var ctorSig, ctorArgs, setup, cleanup: string + var ctorSig, ctorArgs, decls, setup, cleanup: string for (fieldName, fieldType) in fields: let t = jtype(fieldType) ctorSig.add t.publicSig case t.kind of kPrim: if t.java == "boolean": - ctorArgs.add &", (jboolean)(v.{fieldName} ? JNI_TRUE : JNI_FALSE)" + ctorArgs.add &", (jboolean)(v.f_{fieldName} ? JNI_TRUE : JNI_FALSE)" else: - ctorArgs.add &", ({t.jni})v.{fieldName}" + ctorArgs.add &", ({t.jni})v.f_{fieldName}" of kEnum: - setup.add &" jobject f_{fieldName} = (*env)->CallStaticObjectMethod(env, genny_class_{t.java}, genny_from_ordinal_{t.java}, (jint)v.{fieldName});\n" - cleanup.add &" (*env)->DeleteLocalRef(env, f_{fieldName});\n" - ctorArgs.add &", f_{fieldName}" + decls.add &" jobject j_{fieldName} = NULL;\n" + setup.add &" j_{fieldName} = (*env)->CallStaticObjectMethod(env, genny_class_{t.java}, genny_from_value_{t.java}, (jint)v.f_{fieldName});\n" + setup.add " if ((*env)->ExceptionCheck(env)) goto genny_done;\n" + cleanup.add &" if (j_{fieldName} != NULL) (*env)->DeleteLocalRef(env, j_{fieldName});\n" + ctorArgs.add &", j_{fieldName}" of kValue: - setup.add &" jobject f_{fieldName} = genny_from_{t.c}(env, v.{fieldName});\n" - cleanup.add &" (*env)->DeleteLocalRef(env, f_{fieldName});\n" - ctorArgs.add &", f_{fieldName}" + decls.add &" jobject j_{fieldName} = NULL;\n" + setup.add &" j_{fieldName} = genny_from_{t.c}(env, v.f_{fieldName});\n" + setup.add &" if (j_{fieldName} == NULL) goto genny_done;\n" + cleanup.add &" if (j_{fieldName} != NULL) (*env)->DeleteLocalRef(env, j_{fieldName});\n" + ctorArgs.add &", j_{fieldName}" else: discard cHelpers.add &"static jobject genny_from_{name}(JNIEnv *env, {name} v) {{\n" + cHelpers.add " jobject r = NULL;\n" + cHelpers.add decls cHelpers.add setup - cHelpers.add &" jobject r = (*env)->NewObject(env, genny_class_{name}, genny_init_{name}{ctorArgs});\n" + cHelpers.add &" r = (*env)->NewObject(env, genny_class_{name}, genny_init_{name}{ctorArgs});\n" + cHelpers.add " if ((*env)->ExceptionCheck(env)) r = NULL;\n" + if setup != "": + cHelpers.add "genny_done:\n" cHelpers.add cleanup cHelpers.add " return r;\n}\n\n" @@ -366,53 +566,70 @@ proc emitValueConversions(name: string, fields: seq[(string, NimNode)]) = cOnLoad.add &" if (genny_init_{name} == NULL) return JNI_ERR;\n" for (fieldName, fieldType) in fields: let t = jtype(fieldType) - cOnLoad.add &" genny_field_{name}_{fieldName} = (*env)->GetFieldID(env, genny_class_{name}, \"{toVarCase(toCamelCase(fieldName))}\", \"{t.publicSig}\");\n" + cOnLoad.add &" genny_field_{name}_{fieldName} = (*env)->GetFieldID(env, genny_class_{name}, \"{javaName(fieldName)}\", \"{t.publicSig}\");\n" cOnLoad.add &" if (genny_field_{name}_{fieldName} == NULL) return JNI_ERR;\n" proc exportObjectJava*(sym: NimNode, constructor: NimNode) = let name = sym.repr + let problem = valueLayoutProblem(sym) + if problem != "": + warning(&"genny java: skipping object {name}, {problem}", sym) + return var fields: seq[(string, NimNode)] supported = true - for identDefs in sym.getImpl()[2][2]: - for property in identDefs[0 .. ^3]: - let t = jtype(identDefs[^2]) - if t.kind notin {kPrim, kEnum, kValue}: - supported = false - fields.add((toSnakeCase(property[1].repr), identDefs[^2])) - if not supported: + if sym.getImpl()[2][2].kind == nnkRecList: + for identDefs in sym.getImpl()[2][2]: + for property in identDefs[0 .. ^3]: + let t = jtype(identDefs[^2]) + if t.kind notin {kPrim, kEnum, kValue}: + supported = false + fields.add((toSnakeCase(property.fieldIdent.repr), identDefs[^2])) + if not supported or fields.len == 0: warning(&"genny java: skipping object {name}, only primitive, enum and object fields are supported", sym) return valueNames.incl name - valueFields[name] = fields classKinds[name] = kValue emitValueConversions(name, fields) + discard claim(name, "equals", @["java.lang.Object"]) + discard claim(name, "hashCode", @[]) var code = "" - var ctorParams, assigns, eqs, hashes: seq[string] + var ctorParams, defaults, assigns, eqs, hashes: seq[string] for (fieldName, fieldType) in fields: let t = jtype(fieldType) - jname = toVarCase(toCamelCase(fieldName)) + jname = javaName(fieldName) code.add &" public {t.java} {jname};\n" ctorParams.add &"{t.java} {jname}" assigns.add &" this.{jname} = {jname};\n" - if t.kind == kPrim: + case t.kind + of kValue: + defaults.add &" this.{jname} = new {t.java}();\n" + eqs.add &"java.util.Objects.equals({jname}, o.{jname})" + of kEnum: + defaults.add &" this.{jname} = {t.java}.{enumEntries[t.java][0][0]};\n" eqs.add &"{jname} == o.{jname}" else: - eqs.add &"java.util.Objects.equals({jname}, o.{jname})" + case t.java + of "float": eqs.add &"java.lang.Float.compare({jname}, o.{jname}) == 0" + of "double": eqs.add &"java.lang.Double.compare({jname}, o.{jname}) == 0" + else: eqs.add &"{jname} == o.{jname}" hashes.add jname - code.add &"\n public {name}() {{}}\n\n" + code.add &"\n public {name}() {{\n" + code.add defaults.join("") + code.add " }\n\n" code.add &" public {name}({ctorParams.join(\", \")}) {{\n" code.add assigns.join("") code.add " }\n\n" - code.add " @Override public boolean equals(Object other) {\n" + code.add " @java.lang.Override public boolean equals(java.lang.Object other) {\n" + code.add " if (this == other) return true;\n" code.add &" if (!(other instanceof {name})) return false;\n" code.add &" {name} o = ({name})other;\n" code.add &" return {eqs.join(\" && \")};\n" code.add " }\n\n" - code.add " @Override public int hashCode() {\n" + code.add " @java.lang.Override public int hashCode() {\n" code.add &" return java.util.Objects.hash({hashes.join(\", \")});\n" code.add " }\n\n" addMember(name, code) @@ -421,45 +638,87 @@ proc exportObjectJava*(sym: NimNode, constructor: NimNode) = let ctorType = constructor.getTypeInst() ret = jtype(sym) - var params: seq[(string, JType)] + var + params: seq[(string, JType)] + supported = true for param in ctorType[0][1 .. ^1]: - params.add((param[0].repr.split('`')[0], jtype(param[1]))) - let nativeName = emitNative(&"$lib_{toSnakeCase(name)}", params, ret, false) - var publicParams, args: seq[string] + let t = jtype(param[1]) + supported = supported and t.kind != kUnsupported + params.add((param[0].repr.split('`')[0], t)) + let methodName = javaMethodName(constructor) + if not supported or methodName == "": + warning(&"genny java: skipping constructor of {name}, unsupported types", sym) + return + var publicParams, publicTypes, args, fences: seq[string] for (pname, t) in params: - let jname = toVarCase(toCamelCase(pname)) + let jname = javaName(pname) publicParams.add &"{t.java} {jname}" + publicTypes.add t.java args.add javaToNative(t, jname) - var ctor = &" public static {name} {constructor.repr}({publicParams.join(\", \")}) {{\n" - ctor.add javaCall(nativeName, args, ret) - ctor.add " }\n\n" - addMember(name, ctor) + if t.kind == kRef: + fences.add jname + if not claim(name, methodName, publicTypes): + error(&"genny java: constructor {constructor.repr} has the same Java signature as another method of {name}", sym) + let nativeName = emitNative(&"$lib_{toSnakeCase(name)}", params, ret, false) + addMember(name, javaMethod( + &"public static {name} {methodName}({publicParams.join(\", \")})", + nativeName, args, ret, fences)) proc emitHandleClass(name, unrefApi: string) = ## Members shared by ref objects and seqs: the handle, wrapping and - ## releasing. The Cleaner releases on its own thread, and Nim refcounts are - ## not atomic, so objects shared with other threads should be closed - ## explicitly instead of left to the Cleaner. + ## releasing. The Cleaner only queues the release; it runs on the next + ## call into the library, so Nim refcounts are only touched by threads + ## that call the library. let unrefNative = emitNative(unrefApi, @[("x", JType(kind: kRef, native: "long", jni: "jlong", sig: "J", c: "void *"))], jtype(newEmptyNode()), false) + discard claim(name, "close", @[]) var code = "" - code.add " final long handle;\n" - code.add " private final java.lang.ref.Cleaner.Cleanable cleanable;\n\n" - code.add &" private {name}(long handle, Void wrapped) {{\n" - code.add " this.handle = handle;\n" - code.add &" this.cleanable = {libClass}.CLEANER.register(this, new Release(handle));\n" + code.add " private long handle$;\n" + code.add " private final java.lang.ref.Cleaner.Cleanable cleanable$;\n\n" + code.add &" private {name}(long handle, java.lang.Void unused) {{\n" + code.add &" if (handle == 0L) throw new java.lang.IllegalStateException(\"{name} is nil\");\n" + code.add " this.handle$ = handle;\n" + code.add &" this.cleanable$ = {libClass}.CLEANER$.register(this, new {libClass}.Release$(handle, {libClass}::{unrefNative}));\n" code.add " }\n\n" - code.add &" static {name} wrap(long handle) {{\n" + code.add &" static {name} wrap$(long handle) {{\n" code.add &" return handle == 0L ? null : new {name}(handle, null);\n" code.add " }\n\n" - code.add " private static final class Release implements Runnable {\n" - code.add " private final long handle;\n" - code.add " Release(long handle) { this.handle = handle; }\n" - code.add &" @Override public void run() {{ {libClass}.{unrefNative}(handle); }}\n" + code.add " long handle$() {\n" + code.add &" if (handle$ == 0L) throw new java.lang.IllegalStateException(\"{name} is closed\");\n" + code.add " return handle$;\n" + code.add " }\n\n" + code.add " /** Releases the Nim object. Using this object afterwards throws IllegalStateException. */\n" + code.add " @java.lang.Override public void close() {\n" + code.add " if (handle$ == 0L) return;\n" + code.add " handle$ = 0L;\n" + code.add " cleanable$.clean();\n" + code.add &" {libClass}.releasePending();\n" + code.add " }\n\n" + addMember(name, code) + +proc emitConstructor(name, apiName: string, params: seq[(string, JType)], raises: bool) = + ## Public Java constructor backed by a Nim proc returning a new handle. + var publicParams, publicTypes, args, fences: seq[string] + for (pname, t) in params: + let jname = javaName(pname) + publicParams.add &"{t.java} {jname}" + publicTypes.add t.java + args.add javaToNative(t, jname) + if t.kind == kRef: + fences.add jname + if not claim(name, "create$", publicTypes): + error(&"genny java: {name} has more than one constructor") + let nativeName = emitNative(apiName, params, JType(kind: kRef, native: "long", + jni: "jlong", sig: "J", c: "void *"), raises) + var names: seq[string] + for (pname, _) in params: + names.add javaName(pname) + var code = &" public {name}({publicParams.join(\", \")}) {{\n" + code.add &" this(create$({names.join(\", \")}), null);\n" code.add " }\n\n" - code.add " /** Releases the Nim object. Do not use this object afterwards. */\n" - code.add " @Override public void close() { cleanable.clean(); }\n\n" + code.add javaMethod(&"private static long create$({publicParams.join(\", \")})", + nativeName, args, JType(kind: kPrim, java: "long"), fences) addMember(name, code) proc emitSeqAccessors(className, prefix, methodPrefix: string, selfType, entry: JType) = @@ -468,25 +727,37 @@ proc emitSeqAccessors(className, prefix, methodPrefix: string, selfType, entry: index = jtype(ident"int") voidT = jtype(newEmptyNode()) self = ("s", selfType) + entryFences = if entry.kind == kRef: @["this", "v"] else: @["this"] proc name(op: string): string = if methodPrefix == "": op else: methodPrefix & op.capitalizeAscii() + proc claimed(op: string, params: seq[string]): string = + result = name(op) + if not claim(className, result, params): + error(&"genny java: {className}.{result} is generated for a seq but is already taken") + let + lenNative = emitNative(prefix & "_len", @[self], index, false) + bounds = @[ + &"long length$ = {libClass}.{lenNative}(this.handle$());", + "if (i < 0 || i >= length$) throw new java.lang.IndexOutOfBoundsException(\"Index \" + i + \" out of bounds for length \" + length$);" + ] var code = "" - var n = emitNative(prefix & "_len", @[self], index, false) - code.add &" public long {name(\"len\")}() {{\n return {libClass}.{n}(handle);\n }}\n\n" - n = emitNative(prefix & "_get", @[self, ("i", index)], entry, false) - let getCall = nativeToJava(entry, libClass & "." & n & "(handle, i)") - code.add &" public {entry.java} {name(\"get\")}(long i) {{\n" - code.add &" return {getCall};\n }}\n\n" + code.add javaMethod(&"public long {claimed(\"len\", @[])}()", lenNative, + @["this.handle$()"], index, @["this"]) + var n = emitNative(prefix & "_get", @[self, ("i", index)], entry, false) + code.add javaMethod(&"public {entry.java} {claimed(\"get\", @[\"long\"])}(long i)", n, + @["this.handle$()", "i"], entry, @["this"], bounds) n = emitNative(prefix & "_set", @[self, ("i", index), ("v", entry)], voidT, false) - code.add &" public void {name(\"set\")}(long i, {entry.java} v) {{\n" - code.add &" {libClass}.{n}(handle, i, {javaToNative(entry, \"v\")});\n }}\n\n" + code.add javaMethod(&"public void {claimed(\"set\", @[\"long\", entry.java])}(long i, {entry.java} v)", n, + @["this.handle$()", "i", javaToNative(entry, "v")], voidT, entryFences, bounds) n = emitNative(prefix & "_add", @[self, ("v", entry)], voidT, false) - code.add &" public void {name(\"add\")}({entry.java} v) {{\n" - code.add &" {libClass}.{n}(handle, {javaToNative(entry, \"v\")});\n }}\n\n" + code.add javaMethod(&"public void {claimed(\"add\", @[entry.java])}({entry.java} v)", n, + @["this.handle$()", javaToNative(entry, "v")], voidT, entryFences) n = emitNative(prefix & "_delete", @[self, ("i", index)], voidT, false) - code.add &" public void {name(\"delete\")}(long i) {{\n {libClass}.{n}(handle, i);\n }}\n\n" + code.add javaMethod(&"public void {claimed(\"delete\", @[\"long\"])}(long i)", n, + @["this.handle$()", "i"], voidT, @["this"], bounds) n = emitNative(prefix & "_clear", @[self], voidT, false) - code.add &" public void {name(\"clear\")}() {{\n {libClass}.{n}(handle);\n }}\n\n" + code.add javaMethod(&"public void {claimed(\"clear\", @[])}()", n, + @["this.handle$()"], voidT, @["this"]) addMember(className, code) proc exportRefObjectJava*( @@ -510,45 +781,44 @@ proc exportRefObjectJava*( let t = jtype(param[1]) supported = supported and t.kind != kUnsupported params.add((param[0].repr.split('`')[0], t)) - if supported: - let nativeName = emitNative(&"$lib_{toSnakeCase(constructor.repr)}", - params, self, constructor.raises()) - var publicParams, args: seq[string] - for (pname, t) in params: - let jname = toVarCase(toCamelCase(pname)) - publicParams.add &"{t.java} {jname}" - args.add javaToNative(t, jname) - var code = &" public {name}({publicParams.join(\", \")}) {{\n" - code.add &" this({libClass}.{nativeName}({args.join(\", \")}), null);\n" - code.add " }\n\n" - addMember(name, code) - else: + var publicTypes: seq[string] + for (_, t) in params: + publicTypes.add t.java + if not supported: warning(&"genny java: skipping constructor of {name}, unsupported types", sym) + elif publicTypes == @["long", "java.lang.Void"]: + error(&"genny java: constructor of {name} clashes with the internal constructor", sym) + else: + emitConstructor(name, &"$lib_{toSnakeCase(constructor.repr)}", params, + constructor.raises()) for (fieldName, fieldType) in fields: let fieldSnaked = toSnakeCase(fieldName) if fieldType.kind == nnkBracketExpr: let entry = jtype(fieldType[1]) - if entry.kind == kUnsupported or entry.kind == kString: + if entry.kind in {kUnsupported, kString, kBuffer}: warning(&"genny java: skipping field {name}.{fieldName}, unsupported seq entry type", sym) continue emitSeqAccessors(name, &"$lib_{snaked}_{fieldSnaked}", fieldName, self, entry) continue let t = jtype(fieldType) - if t.kind == kUnsupported: + if t.kind in {kUnsupported, kBuffer}: warning(&"genny java: skipping field {name}.{fieldName}, unsupported type", sym) continue + let + cap = fieldName.capitalizeAscii() + fences = if t.kind == kRef: @["this", "v"] else: @["this"] + if not claim(name, "get" & cap, @[]) or not claim(name, "set" & cap, @[t.java]): + error(&"genny java: accessors of {name}.{fieldName} clash with another method", sym) let getter = emitNative(&"$lib_{snaked}_get_{fieldSnaked}", @[("o", self)], t, false) setter = emitNative(&"$lib_{snaked}_set_{fieldSnaked}", @[("o", self), ("v", t)], jtype(newEmptyNode()), false) - cap = fieldName.capitalizeAscii() var code = "" - let getCall = nativeToJava(t, libClass & "." & getter & "(handle)") - code.add &" public {t.java} get{cap}() {{\n" - code.add &" return {getCall};\n }}\n\n" - code.add &" public void set{cap}({t.java} v) {{\n" - code.add &" {libClass}.{setter}(handle, {javaToNative(t, \"v\")});\n }}\n\n" + code.add javaMethod(&"public {t.java} get{cap}()", getter, + @["this.handle$()"], t, @["this"]) + code.add javaMethod(&"public void set{cap}({t.java} v)", setter, + @["this.handle$()", javaToNative(t, "v")], jtype(newEmptyNode()), fences) addMember(name, code) proc exportSeqJava*(sym: NimNode) = @@ -561,13 +831,110 @@ proc exportSeqJava*(sym: NimNode) = self = jtype(sym) entry = jtype(sym[1]) emitHandleClass(name, &"$lib_{snaked}_unref") - let newNative = emitNative(&"$lib_new_{snaked}", @[], self, false) - addMember(name, &" public {name}() {{\n this({libClass}.{newNative}(), null);\n }}\n\n") - if entry.kind == kUnsupported: + emitConstructor(name, &"$lib_new_{snaked}", @[], false) + if entry.kind in {kUnsupported, kBuffer}: warning(&"genny java: skipping accessors of {name}, unsupported entry type", sym) return emitSeqAccessors(name, &"$lib_{snaked}", "", self, entry) +const stringHelpers = """ +static char *genny_utf8_from_jstring(JNIEnv *env, jstring s) { + jsize n = s == NULL ? 0 : (*env)->GetStringLength(env, s); + char *out = (char *)malloc((size_t)n * 3 + 1); + size_t k = 0; + if (out == NULL) { + genny_throw(env, "java/lang/OutOfMemoryError", "genny: string conversion"); + return NULL; + } + if (n > 0) { + const jchar *u = (*env)->GetStringCritical(env, s, NULL); + jsize i; + if (u == NULL) { + free(out); + if (!(*env)->ExceptionCheck(env)) genny_throw(env, "java/lang/OutOfMemoryError", "genny: string conversion"); + return NULL; + } + for (i = 0; i < n; i++) { + uint32_t c = u[i]; + if (c >= 0xD800 && c < 0xDC00 && i + 1 < n && u[i + 1] >= 0xDC00 && u[i + 1] < 0xE000) { + c = 0x10000 + ((c - 0xD800) << 10) + (u[i + 1] - 0xDC00); + i++; + } else if (c >= 0xD800 && c < 0xE000) { + c = 0xFFFD; + } + if (c < 0x80) { + out[k++] = (char)c; + } else if (c < 0x800) { + out[k++] = (char)(0xC0 | (c >> 6)); + out[k++] = (char)(0x80 | (c & 0x3F)); + } else if (c < 0x10000) { + out[k++] = (char)(0xE0 | (c >> 12)); + out[k++] = (char)(0x80 | ((c >> 6) & 0x3F)); + out[k++] = (char)(0x80 | (c & 0x3F)); + } else { + out[k++] = (char)(0xF0 | (c >> 18)); + out[k++] = (char)(0x80 | ((c >> 12) & 0x3F)); + out[k++] = (char)(0x80 | ((c >> 6) & 0x3F)); + out[k++] = (char)(0x80 | (c & 0x3F)); + } + } + (*env)->ReleaseStringCritical(env, s, u); + } + out[k] = 0; + return out; +} + +static jstring genny_jstring_from_utf8(JNIEnv *env, const char *s) { + jchar small[256]; + jchar *buf = small; + const unsigned char *p = (const unsigned char *)(s == NULL ? "" : s); + size_t len = strlen((const char *)p), i = 0, k = 0; + jstring r; + if (len > sizeof(small) / sizeof(small[0])) { + buf = (jchar *)malloc(len * sizeof(jchar)); + if (buf == NULL) { + genny_throw(env, "java/lang/OutOfMemoryError", "genny: string conversion"); + return NULL; + } + } + while (i < len) { + uint32_t c = p[i]; + int extra = c < 0x80 ? 0 : c < 0xC2 ? -1 : c < 0xE0 ? 1 : c < 0xF0 ? 2 : c < 0xF5 ? 3 : -1; + int j; + if (extra > 0) { + if (i + (size_t)extra >= len) { + extra = -1; + } else { + c &= 0x3F >> extra; + for (j = 1; j <= extra; j++) { + if ((p[i + j] & 0xC0) != 0x80) { extra = -1; break; } + c = (c << 6) | (p[i + j] & 0x3F); + } + if (extra == 2 && (c < 0x800 || (c >= 0xD800 && c < 0xE000))) extra = -1; + if (extra == 3 && (c < 0x10000 || c > 0x10FFFF)) extra = -1; + } + } + if (extra < 0) { + buf[k++] = 0xFFFD; + i++; + continue; + } + i += (size_t)extra + 1; + if (c >= 0x10000) { + c -= 0x10000; + buf[k++] = (jchar)(0xD800 + (c >> 10)); + buf[k++] = (jchar)(0xDC00 + (c & 0x3FF)); + } else { + buf[k++] = (jchar)c; + } + } + r = (*env)->NewString(env, buf, (jsize)k); + if (buf != small) free(buf); + return r; +} + +""" + proc writeJava*(dir, lib: string): NimNode = let className = javaClassName(lib) @@ -575,9 +942,14 @@ proc writeJava*(dir, lib: string): NimNode = cPath = dir / (toSnakeCase(lib) & "_jni.c") javaHome = getEnv("JAVA_HOME") exceptionClass = className & "Exception" + # With --app:lib Nim initializes itself when the library is loaded. + nimInitializes = appType == "lib" and not compileOption("noMain") if javaHome == "": error("genny java: JAVA_HOME must be set to a JDK so jni.h can be found") + for name in classKinds.keys: + if name in [className, exceptionClass]: + error(&"genny java: type {name} clashes with the generated {name}.java") createDir(javaDir) @@ -587,27 +959,54 @@ proc writeJava*(dir, lib: string): NimNode = var java = header() java.add &"public final class {className} {{\n" java.add &" private {className}() {{}}\n\n" - java.add " static final java.lang.ref.Cleaner CLEANER = java.lang.ref.Cleaner.create();\n\n" + java.add " static final java.lang.ref.Cleaner CLEANER$ = java.lang.ref.Cleaner.create();\n" + java.add &" private static final java.util.concurrent.ConcurrentLinkedQueue PENDING$ =\n" + java.add " new java.util.concurrent.ConcurrentLinkedQueue<>();\n\n" + java.add " static final class Release$ implements java.lang.Runnable {\n" + java.add " private final long handle;\n" + java.add " private final java.util.function.LongConsumer unref;\n\n" + java.add " Release$(long handle, java.util.function.LongConsumer unref) {\n" + java.add " this.handle = handle;\n" + java.add " this.unref = unref;\n" + java.add " }\n\n" + java.add " @java.lang.Override public void run() { PENDING$.add(this); }\n" + java.add " }\n\n" java.add " /** Loads the native library from an absolute path. */\n" - java.add " public static void load(String absolutePath) {\n" - java.add " System.load(absolutePath);\n" + java.add " public static void load(java.lang.String absolutePath) {\n" + java.add " java.lang.System.load(absolutePath);\n" + java.add " }\n\n" + java.add " /** Releases Nim objects whose Java wrappers were garbage collected. Every call into the library does this first. */\n" + java.add " public static void releasePending() {\n" + java.add " Release$ r;\n" + java.add " while ((r = PENDING$.poll()) != null) r.unref.accept(r.handle);\n" java.add " }\n\n" java.add libMembers java.add "}\n" writeFile(javaDir / (className & ".java"), java.replace(libClass, className)) var exc = header() - exc.add &"public class {exceptionClass} extends RuntimeException {{\n" - exc.add &" public {exceptionClass}(String message) {{ super(message); }}\n" + exc.add &"public class {exceptionClass} extends java.lang.RuntimeException {{\n" + exc.add " private static final long serialVersionUID = 1L;\n\n" + exc.add &" public {exceptionClass}(java.lang.String message) {{ super(message); }}\n" exc.add "}\n" writeFile(javaDir / (exceptionClass & ".java"), exc) - for name, values in enumValues: + for name, entries in enumEntries: var code = header() + var values, cases: seq[string] + for (entry, value) in entries: + values.add &"{entry}({value})" + cases.add &" case {value}: return {entry};\n" code.add &"public enum {name} {{\n" code.add " " & values.join(", ") & ";\n\n" - code.add &" private static final {name}[] VALUES = values();\n\n" - code.add &" static {name} fromOrdinal(int ordinal) {{ return VALUES[ordinal]; }}\n" + code.add " public final int value;\n\n" + code.add &" {name}(int value) {{ this.value = value; }}\n\n" + code.add &" static {name} fromValue(int value) {{\n" + code.add " switch (value) {\n" + code.add cases.join("") + code.add &" default: throw new java.lang.IllegalArgumentException(\"{name} has no value \" + value);\n" + code.add " }\n" + code.add " }\n" code.add "}\n" writeFile(javaDir / (name & ".java"), code) @@ -617,7 +1016,7 @@ proc writeJava*(dir, lib: string): NimNode = of kValue: code.add &"public final class {name} {{\n" else: - code.add &"public final class {name} implements AutoCloseable {{\n" + code.add &"public final class {name} implements java.lang.AutoCloseable {{\n" code.add members code.add "}\n" writeFile(javaDir / (name & ".java"), code.replace(libClass, className)) @@ -626,16 +1025,19 @@ proc writeJava*(dir, lib: string): NimNode = code.add "#include \n" code.add "#include \n" code.add "#include \n" + code.add "#include \n" code.add "#include \n\n" - code.add "extern void NimMain(void);\n\n" + if not nimInitializes: + code.add "extern void NimMain(void);\n\n" if needsErrorBridge: code.add "extern bool $lib_check_error(void);\n" code.add "extern char *$lib_take_error(void);\n" code.add "static jclass genny_exception_class;\n" - code.add "static jmethodID genny_enum_ordinal;\n" - for name in enumNames: + code.add "static jmethodID genny_exception_init;\n" + for name in enumEntries.keys: code.add &"static jclass genny_class_{name};\n" - code.add &"static jmethodID genny_from_ordinal_{name};\n" + code.add &"static jmethodID genny_from_value_{name};\n" + code.add &"static jfieldID genny_enum_value_{name};\n" code.add "\n" code.add "static int genny_load_class(JNIEnv *env, const char *name, jclass *out) {\n" code.add " jclass local = (*env)->FindClass(env, name);\n" @@ -644,6 +1046,26 @@ proc writeJava*(dir, lib: string): NimNode = code.add " (*env)->DeleteLocalRef(env, local);\n" code.add " return *out != NULL;\n" code.add "}\n\n" + code.add "static void genny_throw(JNIEnv *env, const char *name, const char *message) {\n" + code.add " jclass c = (*env)->FindClass(env, name);\n" + code.add " if (c == NULL) return;\n" + code.add " (*env)->ThrowNew(env, c, message);\n" + code.add " (*env)->DeleteLocalRef(env, c);\n" + code.add "}\n\n" + if needsStrings: + code.add stringHelpers + if needsErrorBridge: + code.add "static void genny_raise(JNIEnv *env) {\n" + code.add " char *message = $lib_take_error();\n" + code.add " jstring jmessage = genny_jstring_from_utf8(env, message == NULL ? \"error\" : message);\n" + code.add " jthrowable e;\n" + code.add " if (jmessage == NULL) return;\n" + code.add " e = (jthrowable)(*env)->NewObject(env, genny_exception_class, genny_exception_init, jmessage);\n" + code.add " (*env)->DeleteLocalRef(env, jmessage);\n" + code.add " if ((*env)->ExceptionCheck(env)) return;\n" + code.add " (*env)->Throw(env, e);\n" + code.add " (*env)->DeleteLocalRef(env, e);\n" + code.add "}\n\n" code.add cTypes code.add cProtos code.add "\n" @@ -653,35 +1075,44 @@ proc writeJava*(dir, lib: string): NimNode = code.add cMethods code.add "};\n\n" code.add "JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {\n" - code.add " (void)reserved;\n" code.add " JNIEnv *env;\n" + code.add " jclass cls;\n" + code.add " jint registered;\n" + code.add " (void)reserved;\n" code.add " if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_8) != JNI_OK) return JNI_ERR;\n" - code.add " NimMain();\n" + if not nimInitializes: + code.add " NimMain();\n" code.add &" if (!genny_load_class(env, \"{classPath(exceptionClass)}\", &genny_exception_class)) return JNI_ERR;\n" - code.add " {\n" - code.add " jclass enumClass = (*env)->FindClass(env, \"java/lang/Enum\");\n" - code.add " if (enumClass == NULL) return JNI_ERR;\n" - code.add " genny_enum_ordinal = (*env)->GetMethodID(env, enumClass, \"ordinal\", \"()I\");\n" - code.add " (*env)->DeleteLocalRef(env, enumClass);\n" - code.add " }\n" - for name in enumNames: + code.add " genny_exception_init = (*env)->GetMethodID(env, genny_exception_class, \"\", \"(Ljava/lang/String;)V\");\n" + code.add " if (genny_exception_init == NULL) return JNI_ERR;\n" + for name in enumEntries.keys: code.add &" if (!genny_load_class(env, \"{classPath(name)}\", &genny_class_{name})) return JNI_ERR;\n" - code.add &" genny_from_ordinal_{name} = (*env)->GetStaticMethodID(env, genny_class_{name}, \"fromOrdinal\", \"(I)L{classPath(name)};\");\n" - code.add &" if (genny_from_ordinal_{name} == NULL) return JNI_ERR;\n" + code.add &" genny_from_value_{name} = (*env)->GetStaticMethodID(env, genny_class_{name}, \"fromValue\", \"(I)L{classPath(name)};\");\n" + code.add &" if (genny_from_value_{name} == NULL) return JNI_ERR;\n" + code.add &" genny_enum_value_{name} = (*env)->GetFieldID(env, genny_class_{name}, \"value\", \"I\");\n" + code.add &" if (genny_enum_value_{name} == NULL) return JNI_ERR;\n" code.add cOnLoad - code.add " jclass cls;\n" - code.add &" if (!genny_load_class(env, \"{classPath(className)}\", &cls)) return JNI_ERR;\n" - code.add " jint count = (jint)(sizeof(genny_jni_methods) / sizeof(genny_jni_methods[0]));\n" - code.add " if ((*env)->RegisterNatives(env, cls, genny_jni_methods, count) != 0) return JNI_ERR;\n" + code.add &" cls = (*env)->FindClass(env, \"{classPath(className)}\");\n" + code.add " if (cls == NULL) return JNI_ERR;\n" + code.add " registered = (*env)->RegisterNatives(env, cls, genny_jni_methods,\n" + code.add " (jint)(sizeof(genny_jni_methods) / sizeof(genny_jni_methods[0])));\n" + code.add " (*env)->DeleteLocalRef(env, cls);\n" + code.add " if (registered != 0) return JNI_ERR;\n" code.add " return JNI_VERSION_1_8;\n" code.add "}\n" code = code.replace("$lib", toSnakeCase(lib)) writeFile(cPath, code) + let platformInclude = + when defined(windows): "win32" + elif defined(macosx): "darwin" + elif defined(freebsd): "freebsd" + elif defined(openbsd): "openbsd" + else: "linux" result = newStmtList() let - includeOpt = "-I" & (javaHome / "include") - includePlatformOpt = "-I" & (javaHome / "include" / "linux") + includeOpt = "-I" & quoteShell(javaHome / "include") + includePlatformOpt = "-I" & quoteShell(javaHome / "include" / platformInclude) compilePath = cPath result.add quote do: {.passC: `includeOpt`.} diff --git a/tests/TestJavaMain.java b/tests/TestJavaMain.java index 23a49fd..ed18bc4 100644 --- a/tests/TestJavaMain.java +++ b/tests/TestJavaMain.java @@ -6,12 +6,36 @@ static void check(boolean ok, String what) { System.out.println("ok " + what); } - public static void main(String[] args) { + static void throwsType(Class type, Runnable r, String what) { + try { + r.run(); + } catch (Throwable t) { + check(type.isInstance(t), what + " (got " + t + ")"); + return; + } + check(false, what + " (nothing thrown)"); + } + + static boolean hasMethod(Class c, String name) { + for (java.lang.reflect.Method m : c.getMethods()) { + if (m.getName().equals(name)) return true; + } + return false; + } + + public static void main(String[] args) throws Exception { TestJava.load(new java.io.File(args[0]).getAbsolutePath()); + check(TestJava.getInitCount() == 1, "Nim initializes once"); + check(TestJava.JAVA_LONG_CONST == 123L, "long const"); check(TestJava.JAVA_DOUBLE_CONST == 1.5, "double const"); check(TestJava.JAVA_STRING_CONST.equals("genny"), "string const"); + check(TestJava.JAVA_TYPED_CONST == 42, "typed int32 const"); + check(TestJava.JAVA_UNSIGNED_CONST == -1, "uint32 const keeps its bits"); + check(TestJava.JAVA_BOOL_CONST, "bool const"); + check(TestJava.JAVA_FLOAT_CONST == 0.25f, "float32 const"); + check(TestJava.JAVA_ESCAPED_CONST.equals("a\0b\"\\\né😀"), "escaped string const"); check(TestJava.addInts(40, 2) == 42, "addInts"); check(TestJava.addInts(Long.MAX_VALUE - 1, 1) == Long.MAX_VALUE, "addInts 64-bit"); @@ -23,15 +47,42 @@ public static void main(String[] args) { check(TestJava.echoByte((byte)7) == 7, "echoByte"); check(TestJava.greet("Java").equals("Hello, Java!"), "greet"); check(TestJava.helloWorld().equals("Hello world from Nim!"), "helloWorld"); + check(TestJava.clampTo(-1, 5) == 5, "Java keyword parameter name"); + + String unicode = "Jäva 😀"; + check(TestJava.greet(unicode).equals("Hello, " + unicode + "!"), "non-ASCII string round trip"); + check(TestJava.utf8Len("😀") == 4, "strings reach Nim as standard UTF-8"); + check(TestJava.utf8Len("a\0b") == 1, "embedded NUL truncates at the C boundary"); + check(TestJava.invalidUtf8().equals("a�b"), "invalid UTF-8 from Nim becomes U+FFFD"); for (int i = 0; i < 3; i++) TestJava.bump(); check(TestJava.getCallCount() == 3, "void proc + state"); check(TestJava.nextColor(Color.BLUE) == Color.RED, "enum param and return"); + check(TestJava.nextLevel(Level.LOW) == Level.MID && TestJava.nextLevel(Level.MID) == Level.HIGH, + "enum with explicit values"); + check(Level.HIGH.value == 10, "enum value"); + throwsType(NullPointerException.class, () -> TestJava.nextColor(null), "null enum param"); Tile tile = TestJava.moveTile(new Tile(new Vec(1f, 2f), Color.GREEN, true), 0.5f, -1f); check(tile.pos.x == 1.5f && tile.pos.y == 1f, "nested value object"); check(tile.color == Color.GREEN && tile.solid, "value object enum and bool fields"); + check(new Tile().color == Color.RED && new Tile().pos != null, "value object defaults"); + throwsType(NullPointerException.class, () -> TestJava.moveTile(null, 0f, 0f), "null value object"); + throwsType(NullPointerException.class, + () -> TestJava.moveTile(new Tile(new Vec(), null, false), 0f, 0f), "null enum in value object"); + throwsType(NullPointerException.class, + () -> TestJava.moveTile(new Tile(null, Color.RED, false), 0f, 0f), "null nested value object"); + + Reading reading = TestJava.boostReading(new Reading(Level.LOW, 21)); + check(reading.level == Level.MID && reading.amount == 42, "4-byte enum in value object"); + check(TestJava.sumBig(new Big(1, 2, 3, 4)) == 10, "bycopy value object param"); + Big big = TestJava.makeBig(5); + check(big.a == 5 && big.d == 8, "bycopy value object return"); + + check(new Vec(0f, Float.NaN).equals(new Vec(0f, Float.NaN)), "NaN fields equal themselves"); + check(!new Vec(0f, 0f).equals(new Vec(-0f, 0f)), "0.0 and -0.0 differ like Float.equals"); + check(new Vec(1f, 2f).hashCode() == new Vec(1f, 2f).hashCode(), "value object hashCode"); try (Counter counter = new Counter(10)) { counter.bumpCounter(); @@ -40,12 +91,32 @@ public static void main(String[] args) { counter.setColor(Color.BLUE); check(counter.getColor() == Color.BLUE, "ref object enum field"); check(counter.historyLen() == 2 && counter.historyGet(1) == 12, "ref object seq field"); + throwsType(IndexOutOfBoundsException.class, () -> counter.historyGet(2), "seq field get bounds"); + throwsType(IndexOutOfBoundsException.class, () -> counter.historySet(-1, 0), "seq field set bounds"); + throwsType(IndexOutOfBoundsException.class, () -> counter.historyDelete(5), "seq field delete bounds"); + check(counter.scaled(3) == 36 && counter.scaledInt64(3) == 72, "overloads with the same Java signature"); + check(!hasMethod(Counter.class, "getData") && !hasMethod(Counter.class, "setData"), + "pointer fields are not exported"); + } + + Counter closed = new Counter(1); + closed.close(); + closed.close(); + throwsType(IllegalStateException.class, () -> closed.getCount(), "closed ref object"); + throwsType(IllegalStateException.class, () -> TestJava.sameCount(closed, closed), "closed ref object argument"); + + try (Counter a = TestJava.makeCounter(4); Counter b = new Counter(4)) { + check(TestJava.sameCount(a, b), "ref object arguments"); } try (SeqString w = TestJava.words()) { check(w.len() == 2 && w.get(1).equals("beta"), "seq[string]"); w.add("gamma"); check(w.get(2).equals("gamma"), "seq[string] add"); + throwsType(IndexOutOfBoundsException.class, () -> w.get(3), "seq get bounds"); + throwsType(IndexOutOfBoundsException.class, () -> w.get(-1), "seq negative index"); + throwsType(IndexOutOfBoundsException.class, () -> w.set(3, "x"), "seq set bounds"); + throwsType(IndexOutOfBoundsException.class, () -> w.delete(3), "seq delete bounds"); } check(TestJava.failIfNegative(5) == 5, "raising proc, no error"); @@ -53,17 +124,33 @@ public static void main(String[] args) { TestJava.failIfNegative(-3); check(false, "raising proc throws"); } catch (TestJavaException e) { - check(e.getMessage().equals("negative: -3"), "raising proc throws"); + check(e.getMessage().equals("negative: -3 ✗"), "raising proc throws with UTF-8 message"); } java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocateDirect(16).order(java.nio.ByteOrder.nativeOrder()); TestJava.fillBuffer(buf, 16, 40); check(buf.getInt(0) == 40 && buf.getInt(12) == 43, "direct ByteBuffer as pointer"); + TestJava.fillBuffer(null, 0, 0); + check(true, "null ByteBuffer as nil pointer"); + throwsType(IllegalArgumentException.class, () -> TestJava.fillBuffer(java.nio.ByteBuffer.allocate(16), 16, 40), + "heap ByteBuffer is rejected"); + + long destroyedBefore = TestJava.getDestroyedCount(); + try (Counter c = TestJava.makeCounter(1)) { + check(c.getCount() == 1, "proc returning a ref object"); + } + check(TestJava.getDestroyedCount() == destroyedBefore + 1, "close releases the Nim object"); - for (int i = 0; i < 10000; i++) new Counter(i); - System.gc(); - try { Thread.sleep(200); } catch (InterruptedException e) {} - check(true, "Cleaner releases unclosed ref objects"); + int dropped = 1000; + for (int i = 0; i < dropped; i++) TestJava.makeCounter(i); + long want = destroyedBefore + 1 + dropped; + long deadline = System.nanoTime() + 10_000_000_000L; + while (TestJava.getDestroyedCount() < want && System.nanoTime() < deadline) { + System.gc(); + Thread.sleep(20); + TestJava.releasePending(); + } + check(TestJava.getDestroyedCount() == want, "Cleaner releases unclosed ref objects"); System.out.println("All Java tests passed"); } diff --git a/tests/test_java.nim b/tests/test_java.nim index cb20295..56685b0 100644 --- a/tests/test_java.nim +++ b/tests/test_java.nim @@ -4,11 +4,24 @@ const javaLongConst = 123 javaDoubleConst = 1.5 javaStringConst = "genny" + javaTypedConst = 42'i32 + javaUnsignedConst = 0xFFFF_FFFF'u32 + javaBoolConst = true + javaFloatConst = 0.25'f32 + javaEscapedConst = "a\0b\"\\\né😀" exportConsts: javaLongConst javaDoubleConst javaStringConst + javaTypedConst + javaUnsignedConst + javaBoolConst + javaFloatConst + javaEscapedConst + +var initCount {.noinit.}: int +inc initCount proc addInts(a, b: int): int = a + b @@ -42,6 +55,18 @@ proc bump() = proc getCallCount(): int = callCount +proc getInitCount(): int = + initCount + +proc utf8Len(s: string): int = + s.len + +proc invalidUtf8(): string = + "a\xffb" + +proc clampTo(x, default: int): int = + if x < 0: default else: x + exportProcs: addInts addInt32s @@ -53,6 +78,10 @@ exportProcs: helloWorld bump getCallCount + getInitCount + utf8Len + invalidUtf8 + clampTo type Color = enum @@ -68,17 +97,43 @@ type color*: Color solid*: bool + Level {.size: 4.} = enum + Low = 1 + Mid = "mid" + High = (10, "high") + + Reading = object + level*: Level + amount*: int32 + + Big {.bycopy.} = object + a*, b*, c*, d*: int64 + + Tracker = object + live: bool + Counter* = ref object count*: int color*: Color history*: seq[int] + data*: pointer + tracker: Tracker TestJavaError = object of ValueError -var lastError: ref TestJavaError +var + lastError: ref TestJavaError + destroyedCount: int + +proc `=destroy`(t: Tracker) = + if t.live: + inc destroyedCount + +proc `=copy`(a: var Tracker, b: Tracker) {.error.} exportEnums: Color + Level exportObject Vec: discard @@ -86,6 +141,12 @@ exportObject Vec: exportObject Tile: discard +exportObject Reading: + discard + +exportObject Big: + discard + proc nextColor(c: Color): Color = Color((c.ord + 1) mod 3) @@ -101,15 +162,45 @@ proc bumpCounter(c: Counter) = inc c.count c.history.add c.count +proc scaled(c: Counter, by: int): int = + c.count * by + +proc scaled(c: Counter, by: int64): int = + c.count * by.int * 2 + exportRefObject Counter: fields: count color history + data constructor: newCounter(int) procs: bumpCounter + scaled(Counter, int) + scaled(Counter, int64) + +proc makeCounter(start: int): Counter = + Counter(count: start, tracker: Tracker(live: true)) + +proc getDestroyedCount(): int = + destroyedCount + +proc sameCount(a, b: Counter): bool = + a.count == b.count + +proc nextLevel(l: Level): Level = + if l == Low: Mid else: High + +proc boostReading(r: Reading): Reading = + Reading(level: nextLevel(r.level), amount: r.amount * 2) + +proc sumBig(b: Big): int64 = + b.a + b.b + b.c + b.d + +proc makeBig(x: int64): Big = + Big(a: x, b: x + 1, c: x + 2, d: x + 3) proc words(): seq[string] = @["alpha", "beta"] @@ -131,7 +222,7 @@ proc takeError(): string = proc failIfNegative(x: int): int {.raises: [TestJavaError].} = if x < 0: - raise newException(TestJavaError, "negative: " & $x) + raise newException(TestJavaError, "negative: " & $x & " ✗") x exportProcs: @@ -142,6 +233,13 @@ exportProcs: checkError takeError failIfNegative + makeCounter + getDestroyedCount + sameCount + nextLevel + boostReading + sumBig + makeBig writeFiles("tests/generated_java", "TestJava") From 376e70fc3ed9b91d12b1028a61825e4f329a8192 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 23:10:26 -0400 Subject: [PATCH 07/10] Run the Java tests under -Xcheck:jni and fail on JNI warnings Co-Authored-By: Claude Opus 5.5 --- .github/workflows/java.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index b44bff3..0d014ee 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -17,12 +17,17 @@ jobs: - name: Build standard test DLL with the Java backend run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated/libtest.so tests/test.nim - name: Run Java test against the standard test DLL + shell: bash run: | javac -d tests/generated/classes tests/generated/java/genny/*.java tests/TestJavaStandard.java - java -cp tests/generated/classes TestJavaStandard tests/generated/libtest.so + java -Xcheck:jni -cp tests/generated/classes TestJavaStandard tests/generated/libtest.so 2>&1 | tee java_standard.log + ! grep -n WARNING java_standard.log - name: Build Java test DLL run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated_java/libtest_java.so tests/test_java.nim - name: Compile Java test run: javac -d tests/generated_java/classes tests/generated_java/java/genny/*.java tests/TestJavaMain.java - name: Run Java test - run: java -cp tests/generated_java/classes TestJavaMain tests/generated_java/libtest_java.so + shell: bash + run: | + java -Xcheck:jni -cp tests/generated_java/classes TestJavaMain tests/generated_java/libtest_java.so 2>&1 | tee java_main.log + ! grep -n WARNING java_main.log From 20636c98c0afda8427e4c30454ac7bea7709f156 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sat, 10 Oct 2026 23:10:26 -0400 Subject: [PATCH 08/10] Document the Java backend in the README Co-Authored-By: Claude Opus 5.5 --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index fcad3c9..c684e7b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,26 @@ Node.js | ffi-napi | ✅ | ✅ | ✅ | ✅ | no | C | .h | ✅ | ✅ | ✅ | ✅ | no | Java | JNI | ✅ | ✅ | ✅ | ✅ | ✅ | +## Java + +Build the library with `JAVA_HOME` pointing at a JDK (9 or newer): + +```sh +nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:libpixie.so bindings.nim +javac -d classes bindings/generated/java/genny/*.java +``` + +`-d:noSignalHandler` is required: the JVM uses `SIGSEGV` internally. The package defaults to `genny`; change it with `-d:gennyJavaPackage=com.example.pixie`. Load the library once with `Pixie.load("/absolute/path/libpixie.so")`. + +* Ref objects and seqs are `AutoCloseable`. `close()` releases the Nim object right away; otherwise a `Cleaner` queues the release and it runs at the start of the next call into the library (or `Pixie.releasePending()`). +* Nim's reference counts and the error state of raising procs are not thread-safe: call the library from one thread at a time. +* Unsigned integers keep their bits in the signed Java type of the same size; Nim `int` is a Java `long`. +* Strings are converted to and from UTF-8. A string containing `\0` is cut there, like in the C API. +* `pointer` parameters take direct `ByteBuffer`s (or `null`); Nim gets the buffer's base address in native byte order and must not keep it after the call. Heap buffers throw `IllegalArgumentException`. +* Value objects are copied in and out as C structs. Objects over 24 bytes must be `{.bycopy.}`; inherited, packed and variant objects are skipped with a warning. +* Overloads that end up with the same Java signature get the overload prefix appended, e.g. `scaledInt64`. +* Like the other backends, a ref object built by its exported constructor keeps one extra Nim reference and is never freed. + ## Quest for "nice" language binding. It would be pretty easy to just export a C API from Nim and have users call the ugly C style methods but that is not our goal with Genny. Instead, we try to generate a "nice" API for each language that feels like it was custom-made. This means, where possible, we: From 4ee4ba0f3b3bcbf8b641a699b86489f950a619e9 Mon Sep 17 00:00:00 2001 From: monofuel Date: Sun, 11 Oct 2026 14:17:19 -0400 Subject: [PATCH 09/10] Java backend: opt-in hot reload loader Building the generated JNI C with -DGENNY_JAVA_HOT_RELOAD turns it into a small loader: Nim exports become function pointers that reload(path) binds to a dlopened Nim library, so the Nim code can be swapped while the JVM keeps running. A failed reload keeps the current library. Handles record the library generation and refuse to cross a reload; their releases are dropped instead of reaching the new library. Normal builds keep plain externs and get reload() throwing UnsupportedOperationException. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/java.yml | 15 ++++ .gitignore | 2 + README.md | 22 +++++ src/genny/languages/java.nim | 167 ++++++++++++++++++++++++++++++++--- tests/TestJavaMain.java | 4 + tests/TestJavaReload.java | 78 ++++++++++++++++ tests/test_java_reload.nim | 38 ++++++++ 7 files changed, 314 insertions(+), 12 deletions(-) create mode 100644 tests/TestJavaReload.java create mode 100644 tests/test_java_reload.nim diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 0d014ee..9978482 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -31,3 +31,18 @@ jobs: run: | java -Xcheck:jni -cp tests/generated_java/classes TestJavaMain tests/generated_java/libtest_java.so 2>&1 | tee java_main.log ! grep -n WARNING java_main.log + - name: Build hot reload test libraries and loader + shell: bash + run: | + D=tests/generated_java_reload + nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:$D/libreload_v1.so tests/test_java_reload.nim + nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -d:reloadVersion=2 -o:$D/libreload_v2.so tests/test_java_reload.nim + nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -d:reloadDropSymbol -o:$D/libreload_drop.so tests/test_java_reload.nim + cc -shared -fPIC -DGENNY_JAVA_HOT_RELOAD -I$JAVA_HOME/include -I$JAVA_HOME/include/linux $D/test_reload_jni.c -ldl -o $D/libreload_loader.so + javac -d $D/classes $D/java/genny/*.java tests/TestJavaReload.java + - name: Run Java hot reload test + shell: bash + run: | + D=tests/generated_java_reload + java -Xcheck:jni -cp $D/classes TestJavaReload $D/libreload_loader.so $D/libreload_v1.so $D/libreload_v2.so $D/libreload_drop.so 2>&1 | tee java_reload.log + ! grep -n WARNING java_reload.log diff --git a/.gitignore b/.gitignore index 6869b11..615f77d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ tests/generated/*_jni.c tests/generated/java/ tests/generated_java/ *.class +tests/generated_java_reload/ +tests/generated_java_reload_drop/ diff --git a/README.md b/README.md index c684e7b..056a186 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,28 @@ javac -d classes bindings/generated/java/genny/*.java * Overloads that end up with the same Java signature get the overload prefix appended, e.g. `scaledInt64`. * Like the other backends, a ref object built by its exported constructor keeps one extra Nim reference and is never freed. +### Hot reload (development) + +The JVM cannot unload a JNI library. For development, build the generated C on its own as a small loader with `-DGENNY_JAVA_HOT_RELOAD`; Java loads the loader once and it `dlopen`s the real Nim library, so the Nim code can be rebuilt and swapped while the JVM keeps running. It uses `dlopen`, so it is not available on Windows: + +```sh +nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:build/libpixie-1.so bindings.nim +cc -shared -fPIC -DGENNY_JAVA_HOT_RELOAD -I$JAVA_HOME/include -I$JAVA_HOME/include/linux \ + bindings/generated/pixie_jni.c -ldl -o libpixie_loader.so +``` + +```java +Pixie.load("/abs/libpixie_loader.so"); +Pixie.reload("/abs/build/libpixie-1.so"); // and again after every rebuild +``` + +* Every call throws `IllegalStateException` until the first successful `reload`. `Pixie.hotReloadable()` tells a loader from a normal build, where `reload` throws `UnsupportedOperationException`. +* A reload that cannot open the library or finds a missing export throws `IllegalStateException` and keeps the current library. +* Each successful reload starts a new generation. Ref objects and seqs from an older generation throw `IllegalStateException` when used, and closing or collecting them does nothing; pending releases are dropped. +* Copy each build to a new path before reloading: `dlopen` hands back the already loaded library for the same file. +* Stop every thread that runs Nim code, and every call into the library, before reloading: the old library is unloaded. +* The exported API must stay the same between reloads; regenerate and rebuild the loader (and restart the JVM) when it changes. + ## Quest for "nice" language binding. It would be pretty easy to just export a C API from Nim and have users call the ugly C style methods but that is not our goal with Genny. Instead, we try to generate a "nice" API for each language that feels like it was custom-made. This means, where possible, we: diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index 3375d73..206794e 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -49,7 +49,7 @@ var refNames {.compiletime.}: HashSet[string] javaSigs {.compiletime.}: HashSet[string] cTypes {.compiletime.}: string - cProtos {.compiletime.}: string + cExports {.compiletime.}: OrderedTable[string, (string, string)] cHelpers {.compiletime.}: string cWrappers {.compiletime.}: string cMethods {.compiletime.}: string @@ -219,7 +219,8 @@ proc emitNative( callArgs.add p libMembers.add &" static native {ret.native} {nativeName}({javaParams.join(\", \")});\n" - cProtos.add &"extern {ret.c} {apiName}({cProtoParams.join(\", \")});\n" + if apiName notin cExports: + cExports[apiName] = (ret.c, cProtoParams.join(", ")) var allJniParams = @["JNIEnv *env", "jclass cls"] allJniParams.add jniParams @@ -230,6 +231,12 @@ proc emitNative( else: cWrappers.add &" {ret.jni} genny_ret = 0;\n" cWrappers.add decls cWrappers.add " (void)env; (void)cls;\n" + cWrappers.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" + cWrappers.add " if (!genny_bound) {\n" + cWrappers.add " genny_throw(env, \"java/lang/IllegalStateException\", \"native library not loaded\");\n" + cWrappers.add(if ret.kind == kVoid: " return;\n" else: " return genny_ret;\n") + cWrappers.add " }\n" + cWrappers.add "#endif\n" cWrappers.add setup let call = &"{apiName}({callArgs.join(\", \")})" cWrappers.add " {\n" @@ -675,16 +682,19 @@ proc emitHandleClass(name, unrefApi: string) = discard claim(name, "close", @[]) var code = "" code.add " private long handle$;\n" + code.add " private final int generation$;\n" code.add " private final java.lang.ref.Cleaner.Cleanable cleanable$;\n\n" code.add &" private {name}(long handle, java.lang.Void unused) {{\n" code.add &" if (handle == 0L) throw new java.lang.IllegalStateException(\"{name} is nil\");\n" code.add " this.handle$ = handle;\n" - code.add &" this.cleanable$ = {libClass}.CLEANER$.register(this, new {libClass}.Release$(handle, {libClass}::{unrefNative}));\n" + code.add &" this.generation$ = {libClass}.GENERATION$;\n" + code.add &" this.cleanable$ = {libClass}.CLEANER$.register(this, new {libClass}.Release$(handle, generation$, {libClass}::{unrefNative}));\n" code.add " }\n\n" code.add &" static {name} wrap$(long handle) {{\n" code.add &" return handle == 0L ? null : new {name}(handle, null);\n" code.add " }\n\n" code.add " long handle$() {\n" + code.add &" if (generation$ != {libClass}.GENERATION$) throw new java.lang.IllegalStateException(\"handle from a previous native library generation\");\n" code.add &" if (handle$ == 0L) throw new java.lang.IllegalStateException(\"{name} is closed\");\n" code.add " return handle$;\n" code.add " }\n\n" @@ -951,6 +961,11 @@ proc writeJava*(dir, lib: string): NimNode = if name in [className, exceptionClass]: error(&"genny java: type {name} clashes with the generated {name}.java") + if not claim(libClass, "reload", @["java.lang.String"]) or + not claim(libClass, "hotReloadable", @[]) or + "\"n_genny_reload\"" in cMethods or "\"n_genny_hot_reloadable\"" in cMethods: + error("genny java: an export clashes with the generated reload or hotReloadable") + createDir(javaDir) proc header(): string = @@ -961,15 +976,21 @@ proc writeJava*(dir, lib: string): NimNode = java.add &" private {className}() {{}}\n\n" java.add " static final java.lang.ref.Cleaner CLEANER$ = java.lang.ref.Cleaner.create();\n" java.add &" private static final java.util.concurrent.ConcurrentLinkedQueue PENDING$ =\n" - java.add " new java.util.concurrent.ConcurrentLinkedQueue<>();\n\n" + java.add " new java.util.concurrent.ConcurrentLinkedQueue<>();\n" + java.add " /** Bumped by every reload; handles from an older generation are dead. */\n" + java.add " static volatile int GENERATION$;\n\n" java.add " static final class Release$ implements java.lang.Runnable {\n" java.add " private final long handle;\n" + java.add " private final int generation;\n" java.add " private final java.util.function.LongConsumer unref;\n\n" - java.add " Release$(long handle, java.util.function.LongConsumer unref) {\n" + java.add " Release$(long handle, int generation, java.util.function.LongConsumer unref) {\n" java.add " this.handle = handle;\n" + java.add " this.generation = generation;\n" java.add " this.unref = unref;\n" java.add " }\n\n" - java.add " @java.lang.Override public void run() { PENDING$.add(this); }\n" + java.add " @java.lang.Override public void run() {\n" + java.add " if (generation == GENERATION$) PENDING$.add(this);\n" + java.add " }\n" java.add " }\n\n" java.add " /** Loads the native library from an absolute path. */\n" java.add " public static void load(java.lang.String absolutePath) {\n" @@ -978,8 +999,27 @@ proc writeJava*(dir, lib: string): NimNode = java.add " /** Releases Nim objects whose Java wrappers were garbage collected. Every call into the library does this first. */\n" java.add " public static void releasePending() {\n" java.add " Release$ r;\n" - java.add " while ((r = PENDING$.poll()) != null) r.unref.accept(r.handle);\n" + java.add " while ((r = PENDING$.poll()) != null) {\n" + java.add " if (r.generation == GENERATION$) r.unref.accept(r.handle);\n" + java.add " }\n" + java.add " }\n\n" + java.add " /**\n" + java.add " * Development only: binds the library to the Nim library at absolutePath. Needs a\n" + java.add " * loader built with GENNY_JAVA_HOT_RELOAD, otherwise it throws UnsupportedOperationException.\n" + java.add " * Handles created before a successful reload throw IllegalStateException afterwards.\n" + java.add " * A failed reload throws IllegalStateException and keeps the current library.\n" + java.add " */\n" + java.add " public static synchronized void reload(java.lang.String absolutePath) {\n" + java.add " PENDING$.clear();\n" + java.add " n_genny_reload(absolutePath);\n" + java.add " GENERATION$ = GENERATION$ + 1;\n" + java.add " }\n\n" + java.add " /** True when the loaded library was built with GENNY_JAVA_HOT_RELOAD. */\n" + java.add " public static boolean hotReloadable() {\n" + java.add " return n_genny_hot_reloadable();\n" java.add " }\n\n" + java.add " static native void n_genny_reload(java.lang.String path);\n" + java.add " static native boolean n_genny_hot_reloadable();\n" java.add libMembers java.add "}\n" writeFile(javaDir / (className & ".java"), java.replace(libClass, className)) @@ -1021,17 +1061,47 @@ proc writeJava*(dir, lib: string): NimNode = code.add "}\n" writeFile(javaDir / (name & ".java"), code.replace(libClass, className)) + # Nim exports are plain externs, or with GENNY_JAVA_HOT_RELOAD function + # pointers that n_genny_reload binds to a dlopened Nim library. + var + bound: seq[(string, string, string)] + boundNames: HashSet[string] + proc protos(entries: seq[(string, (string, string))]): string = + var hot, normal: string + for (name, sig) in entries: + let (ret, params) = sig + if name in boundNames: + continue + boundNames.incl name + let fnParams = if params == "": "void" else: params + bound.add((name, ret, fnParams)) + hot.add &"static {ret} (*{name})({fnParams});\n" + normal.add &"extern {ret} {name}({params});\n" + if hot != "": + result = "#ifdef GENNY_JAVA_HOT_RELOAD\n" & hot & "#else\n" & normal & "#endif\n" + var code = "" code.add "#include \n" code.add "#include \n" code.add "#include \n" code.add "#include \n" - code.add "#include \n\n" + code.add "#include \n" + code.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" + code.add "#include \n" + code.add "#include \n\n" + code.add "static void *genny_handle = NULL;\n" + code.add "static int genny_bound = 0;\n" + code.add "#endif\n\n" if not nimInitializes: - code.add "extern void NimMain(void);\n\n" + # The dlopened Nim library of a hot reload build initializes itself. + code.add "#ifndef GENNY_JAVA_HOT_RELOAD\n" + code.add "extern void NimMain(void);\n" + code.add "#endif\n\n" if needsErrorBridge: - code.add "extern bool $lib_check_error(void);\n" - code.add "extern char *$lib_take_error(void);\n" + code.add protos(@[ + ("$lib_check_error", ("bool", "void")), + ("$lib_take_error", ("char *", "void")) + ]) code.add "static jclass genny_exception_class;\n" code.add "static jmethodID genny_exception_init;\n" for name in enumEntries.keys: @@ -1067,12 +1137,83 @@ proc writeJava*(dir, lib: string): NimNode = code.add " (*env)->DeleteLocalRef(env, e);\n" code.add "}\n\n" code.add cTypes - code.add cProtos + var exports: seq[(string, (string, string))] + for name, sig in cExports: + exports.add((name, sig)) + code.add protos(exports) code.add "\n" code.add cHelpers code.add cWrappers + + code.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" + code.add "/* Resolves every export in h; the pointers change only when all are found. */\n" + code.add "static const char *genny_bind(void *h) {\n" + if bound.len == 0: + code.add " (void)h;\n" + else: + code.add " static const char *const names[] = {\n" + for (name, _, _) in bound: + code.add &" \"{name}\",\n" + code.add " };\n" + code.add &" void *syms[{bound.len}];\n" + code.add " size_t i;\n" + code.add " for (i = 0; i < sizeof(names) / sizeof(names[0]); i++) {\n" + code.add " syms[i] = dlsym(h, names[i]);\n" + code.add " if (syms[i] == NULL) return names[i];\n" + code.add " }\n" + for i, (name, ret, params) in bound: + code.add &" {name} = ({ret} (*)({params}))syms[{i}];\n" + code.add " return NULL;\n" + code.add "}\n\n" + code.add "static void JNICALL genny_jni_genny_reload(JNIEnv *env, jclass cls, jstring path) {\n" + code.add " const char *p;\n" + code.add " const char *missing;\n" + code.add " void *h;\n" + code.add " (void)cls;\n" + code.add " if (path == NULL) {\n" + code.add " genny_throw(env, \"java/lang/NullPointerException\", \"path is null\");\n" + code.add " return;\n" + code.add " }\n" + code.add " p = (*env)->GetStringUTFChars(env, path, NULL);\n" + code.add " if (p == NULL) return;\n" + code.add " h = dlopen(p, RTLD_NOW | RTLD_LOCAL);\n" + code.add " (*env)->ReleaseStringUTFChars(env, path, p);\n" + code.add " if (h == NULL) {\n" + code.add " const char *e = dlerror();\n" + code.add " genny_throw(env, \"java/lang/IllegalStateException\", e == NULL ? \"dlopen failed\" : e);\n" + code.add " return;\n" + code.add " }\n" + code.add " missing = genny_bind(h);\n" + code.add " if (missing != NULL) {\n" + code.add " char message[512];\n" + code.add " snprintf(message, sizeof(message), \"native library is missing symbol %s\", missing);\n" + code.add " dlclose(h);\n" + code.add " genny_throw(env, \"java/lang/IllegalStateException\", message);\n" + code.add " return;\n" + code.add " }\n" + code.add " if (genny_handle != NULL) dlclose(genny_handle);\n" + code.add " genny_handle = h;\n" + code.add " genny_bound = 1;\n" + code.add "}\n" + code.add "#else\n" + code.add "static void JNICALL genny_jni_genny_reload(JNIEnv *env, jclass cls, jstring path) {\n" + code.add " (void)cls; (void)path;\n" + code.add " genny_throw(env, \"java/lang/UnsupportedOperationException\", \"built without GENNY_JAVA_HOT_RELOAD\");\n" + code.add "}\n" + code.add "#endif\n\n" + code.add "static jboolean JNICALL genny_jni_genny_hot_reloadable(JNIEnv *env, jclass cls) {\n" + code.add " (void)env; (void)cls;\n" + code.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" + code.add " return JNI_TRUE;\n" + code.add "#else\n" + code.add " return JNI_FALSE;\n" + code.add "#endif\n" + code.add "}\n\n" + code.add "static JNINativeMethod genny_jni_methods[] = {\n" code.add cMethods + code.add " {\"n_genny_reload\", \"(Ljava/lang/String;)V\", (void *)genny_jni_genny_reload},\n" + code.add " {\"n_genny_hot_reloadable\", \"()Z\", (void *)genny_jni_genny_hot_reloadable},\n" code.add "};\n\n" code.add "JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {\n" code.add " JNIEnv *env;\n" @@ -1081,7 +1222,9 @@ proc writeJava*(dir, lib: string): NimNode = code.add " (void)reserved;\n" code.add " if ((*vm)->GetEnv(vm, (void **)&env, JNI_VERSION_1_8) != JNI_OK) return JNI_ERR;\n" if not nimInitializes: + code.add "#ifndef GENNY_JAVA_HOT_RELOAD\n" code.add " NimMain();\n" + code.add "#endif\n" code.add &" if (!genny_load_class(env, \"{classPath(exceptionClass)}\", &genny_exception_class)) return JNI_ERR;\n" code.add " genny_exception_init = (*env)->GetMethodID(env, genny_exception_class, \"\", \"(Ljava/lang/String;)V\");\n" code.add " if (genny_exception_init == NULL) return JNI_ERR;\n" diff --git a/tests/TestJavaMain.java b/tests/TestJavaMain.java index ed18bc4..8d793ed 100644 --- a/tests/TestJavaMain.java +++ b/tests/TestJavaMain.java @@ -27,6 +27,10 @@ public static void main(String[] args) throws Exception { TestJava.load(new java.io.File(args[0]).getAbsolutePath()); check(TestJava.getInitCount() == 1, "Nim initializes once"); + check(!TestJava.hotReloadable(), "normal build is not hot reloadable"); + throwsType(UnsupportedOperationException.class, () -> TestJava.reload("/nonexistent.so"), + "reload on a normal build"); + check(TestJava.addInts(1, 2) == 3, "normal build still works after reload is refused"); check(TestJava.JAVA_LONG_CONST == 123L, "long const"); check(TestJava.JAVA_DOUBLE_CONST == 1.5, "double const"); diff --git a/tests/TestJavaReload.java b/tests/TestJavaReload.java new file mode 100644 index 0000000..6a6bedb --- /dev/null +++ b/tests/TestJavaReload.java @@ -0,0 +1,78 @@ +import genny.*; + +/** + * args: loader.so v1.so v2.so dropped-symbol.so + * The loader is the generated C built with -DGENNY_JAVA_HOT_RELOAD. + */ +public class TestJavaReload { + static void check(boolean ok, String what) { + if (!ok) throw new AssertionError(what); + System.out.println("ok " + what); + } + + static Throwable throwsType(Class type, Runnable r, String what) { + try { + r.run(); + } catch (Throwable t) { + check(type.isInstance(t), what + " (got " + t + ")"); + return t; + } + check(false, what + " (nothing thrown)"); + return null; + } + + static String abs(String path) { + return new java.io.File(path).getAbsolutePath(); + } + + public static void main(String[] args) throws Exception { + TestReload.load(abs(args[0])); + check(TestReload.hotReloadable(), "loader is hot reloadable"); + Throwable t = throwsType(IllegalStateException.class, () -> TestReload.version(), + "calls before the first reload throw"); + check(t.getMessage().equals("native library not loaded"), "not loaded message"); + + TestReload.reload(abs(args[1])); + check(TestReload.version() == 1, "v1 bound"); + check(TestReload.bonus() == 7, "v1 bonus"); + Box box = new Box(5); + check(box.getValue() == 5, "ref object from v1"); + box.setValue(6); + check(box.getValue() == 6, "ref object field from v1"); + + TestReload.reload(abs(args[2])); + check(TestReload.version() == 2, "v2 bound"); + t = throwsType(IllegalStateException.class, () -> box.getValue(), "v1 handle throws after reload"); + check(t.getMessage().equals("handle from a previous native library generation"), "stale handle message"); + box.close(); + check(true, "closing a stale handle does nothing"); + try (Box fresh = new Box(9)) { + check(fresh.getValue() == 9, "ref object from v2"); + } + + for (int i = 0; i < 1000; i++) new Box(i); + TestReload.reload(abs(args[2])); + check(TestReload.version() == 2, "reloading again"); + for (int i = 0; i < 5; i++) { + System.gc(); + Thread.sleep(20); + TestReload.releasePending(); + } + check(TestReload.version() == 2, "dropped stale handles are not released into the new library"); + + throwsType(IllegalStateException.class, () -> TestReload.reload("/nonexistent.so"), + "missing file is rejected"); + check(TestReload.version() == 2, "failed dlopen keeps the old library"); + + t = throwsType(IllegalStateException.class, () -> TestReload.reload(abs(args[3])), + "library missing a symbol is rejected"); + check(t.getMessage().contains("test_reload_bonus"), "missing symbol is named"); + check(TestReload.version() == 2 && TestReload.bonus() == 7, "rejected library keeps the old one"); + try (Box after = new Box(3)) { + check(after.getValue() == 3, "handles still work after a failed reload"); + } + + throwsType(NullPointerException.class, () -> TestReload.reload(null), "null path"); + System.out.println("All Java reload tests passed"); + } +} diff --git a/tests/test_java_reload.nim b/tests/test_java_reload.nim new file mode 100644 index 0000000..ed90448 --- /dev/null +++ b/tests/test_java_reload.nim @@ -0,0 +1,38 @@ +## Library for the Java hot reload test. Built several times with different +## defines; the loader comes from the generated C built with +## -DGENNY_JAVA_HOT_RELOAD. +import genny + +const reloadVersion {.intdefine.} = 1 + +type + Box* = ref object + value*: int + +proc newBox(value: int): Box = + Box(value: value) + +exportRefObject Box: + fields: + value + constructor: + newBox(int) + +proc version(): int = + reloadVersion + +proc bonus(): int = + 7 + +when defined(reloadDropSymbol): + # Same API without `bonus`: a loader built from the full API rejects it. + exportProcs: + version + writeFiles("tests/generated_java_reload_drop", "TestReload") + include generated_java_reload_drop/internal +else: + exportProcs: + version + bonus + writeFiles("tests/generated_java_reload", "TestReload") + include generated_java_reload/internal From b9bb8fea814e283de80a7ed5e46fe0b58cdf0434 Mon Sep 17 00:00:00 2001 From: monofuel Date: Tue, 13 Oct 2026 00:56:24 -0400 Subject: [PATCH 10/10] Make JNI reloads transactional and retain returned strings Validate generated ABI fingerprints before activation and retain the previous library for commit or rollback. Release native references before switching and permanently invalidate stale handles, including after rollback. Keep Nim string results and error messages owned until JNI finishes copying them. Cover preparation, cancellation, rollback and handle generations in the Java reload suite; normal Java and reload tests pass. --- README.md | 22 +++- src/genny/internal.nim | 13 ++- src/genny/languages/java.nim | 212 ++++++++++++++++++++++++----------- src/genny/languages/nim.nim | 5 +- tests/TestJavaReload.java | 21 +++- 5 files changed, 201 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 056a186..bcff13d 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,26 @@ cc -shared -fPIC -DGENNY_JAVA_HOT_RELOAD -I$JAVA_HOME/include -I$JAVA_HOME/inclu bindings/generated/pixie_jni.c -ldl -o libpixie_loader.so ``` +The generated class provides `prepareReload(path)`, `activateReload()`, +`rollbackReload()`, `commitReload()` and `cancelReload()`. Preparation checks +every symbol and the generated ABI fingerprint without changing the active +table. Activation retains the previous table and library handle; commit +unloads the previous library, and rollback restores it after the failed +candidate has been disposed. The caller must quiesce all native calls and +stop every native thread before retiring a generation. Run untrusted candidate +initialization in a separate process before calling prepare. + +`ABI_FINGERPRINT` covers exported signatures, value layouts, ref-object +accessors, enums, constants and the error protocol. Incompatible API changes +are rejected and need new Java bindings. Every activation and rollback +advances the Java handle generation; rollback never revives old handles. +Live and queued native references are released before changing generations. +Nim string results remain owned by a thread-local holder until JNI finishes +copying them, then JNI releases the holder. This includes error messages and +string getters; returned pointers never outlive their Nim storage. +`reload(path)` remains a convenience prepare/activate/commit sequence for +applications that have already stopped their native runtime. + ```java Pixie.load("/abs/libpixie_loader.so"); Pixie.reload("/abs/build/libpixie-1.so"); // and again after every rebuild @@ -63,7 +83,7 @@ Pixie.reload("/abs/build/libpixie-1.so"); // and again after every rebuild * Every call throws `IllegalStateException` until the first successful `reload`. `Pixie.hotReloadable()` tells a loader from a normal build, where `reload` throws `UnsupportedOperationException`. * A reload that cannot open the library or finds a missing export throws `IllegalStateException` and keeps the current library. -* Each successful reload starts a new generation. Ref objects and seqs from an older generation throw `IllegalStateException` when used, and closing or collecting them does nothing; pending releases are dropped. +* Each activation and rollback starts a new generation. Ref objects and seqs from an older generation throw `IllegalStateException` when used. Their references are released before switching; closing or collecting stale handles does nothing. * Copy each build to a new path before reloading: `dlopen` hands back the already loaded library for the same file. * Stop every thread that runs Nim code, and every call into the library, before reloading: the old library is unloaded. * The exported API must stay the same between reloads; regenerate and rebuild the loader (and restart the JVM) when it changes. diff --git a/src/genny/internal.nim b/src/genny/internal.nim index dd2ae54..c3455de 100644 --- a/src/genny/internal.nim +++ b/src/genny/internal.nim @@ -265,7 +265,18 @@ when (NimMajor, NimMinor, NimPatch) == (1, 6, 2): proc writeInternal*(dir, lib: string) = createDir(dir) + let stringHeader = when defined(gennyJava): """ +var gennyJavaReturn {.threadvar.}: string + +proc gennyJavaCString(value: string): cstring = + gennyJavaReturn = value + gennyJavaReturn.cstring + +proc $lib_java_release_string*() {.raises: [], cdecl, exportc, dynlib.} = + gennyJavaReturn = "" + +""" else: "" writeFile( &"{dir}/internal.nim", - header & internal.replace("$Lib", lib).replace("$lib", toSnakeCase(lib)) + (header & stringHeader & internal).replace("$Lib", lib).replace("$lib", toSnakeCase(lib)) ) diff --git a/src/genny/languages/java.nim b/src/genny/languages/java.nim index 206794e..c483ce8 100644 --- a/src/genny/languages/java.nim +++ b/src/genny/languages/java.nim @@ -248,11 +248,8 @@ proc emitNative( case ret.kind of kVoid: discard of kString: - # TODO: under ARC/ORC the char* from the internal export points into a - # Nim string that is already freed when it returns. It is copied here - # before anything else allocates; revisit with a threadvar holder. needsStrings = true - store = "genny_ret = genny_jstring_from_utf8(env, genny_result);" + store = "{ genny_ret = genny_jstring_from_utf8(env, genny_result); $lib_java_release_string(); }" of kPrim: store = if ret.java == "boolean": "genny_ret = genny_result ? JNI_TRUE : JNI_FALSE;" @@ -682,7 +679,7 @@ proc emitHandleClass(name, unrefApi: string) = discard claim(name, "close", @[]) var code = "" code.add " private long handle$;\n" - code.add " private final int generation$;\n" + code.add " private final long generation$;\n" code.add " private final java.lang.ref.Cleaner.Cleanable cleanable$;\n\n" code.add &" private {name}(long handle, java.lang.Void unused) {{\n" code.add &" if (handle == 0L) throw new java.lang.IllegalStateException(\"{name} is nil\");\n" @@ -946,6 +943,18 @@ static jstring genny_jstring_from_utf8(JNIEnv *env, const char *s) { """ proc writeJava*(dir, lib: string): NimNode = + var schema = cTypes & cMethods & libMembers & $needsErrorBridge & "genny-transaction-v2-owned-strings" + schema.add "$lib_java_release_string:void(void)" + for name, signature in cExports: + schema.add name & signature[0] & signature[1] + for name, members in classMembers: + schema.add name & members + for name, entries in enumEntries: + schema.add name & $entries + var fingerprint = 14695981039346656037'u64 + for ch in schema: + fingerprint = (fingerprint xor ch.uint64) * 1099511628211'u64 + let abi = fingerprint.toHex(16) let className = javaClassName(lib) javaDir = dir / "java" / gennyJavaPackage.replace('.', '/') @@ -965,6 +974,11 @@ proc writeJava*(dir, lib: string): NimNode = not claim(libClass, "hotReloadable", @[]) or "\"n_genny_reload\"" in cMethods or "\"n_genny_hot_reloadable\"" in cMethods: error("genny java: an export clashes with the generated reload or hotReloadable") + for phase in ["activate", "rollback", "commit", "cancel"]: + if not claim(libClass, phase & "Reload", @[]): + error("genny java: an export clashes with the generated reload transaction") + if not claim(libClass, "prepareReload", @["java.lang.String"]) or libMembers.contains("ABI_FINGERPRINT"): + error("genny java: an export clashes with the generated reload ABI") createDir(javaDir) @@ -977,19 +991,23 @@ proc writeJava*(dir, lib: string): NimNode = java.add " static final java.lang.ref.Cleaner CLEANER$ = java.lang.ref.Cleaner.create();\n" java.add &" private static final java.util.concurrent.ConcurrentLinkedQueue PENDING$ =\n" java.add " new java.util.concurrent.ConcurrentLinkedQueue<>();\n" + java.add " private static final java.util.Set LIVE$ = java.util.concurrent.ConcurrentHashMap.newKeySet();\n" + java.add " private static boolean PREPARED$, ACTIVE$;\n" java.add " /** Bumped by every reload; handles from an older generation are dead. */\n" - java.add " static volatile int GENERATION$;\n\n" + java.add " static volatile long GENERATION$;\n\n" java.add " static final class Release$ implements java.lang.Runnable {\n" java.add " private final long handle;\n" - java.add " private final int generation;\n" + java.add " private final long generation;\n" java.add " private final java.util.function.LongConsumer unref;\n\n" - java.add " Release$(long handle, int generation, java.util.function.LongConsumer unref) {\n" + java.add " private final java.util.concurrent.atomic.AtomicBoolean released = new java.util.concurrent.atomic.AtomicBoolean();\n" + java.add " Release$(long handle, long generation, java.util.function.LongConsumer unref) {\n" java.add " this.handle = handle;\n" java.add " this.generation = generation;\n" java.add " this.unref = unref;\n" + java.add &" synchronized ({className}.class) {{ LIVE$.add(this); }}\n" java.add " }\n\n" java.add " @java.lang.Override public void run() {\n" - java.add " if (generation == GENERATION$) PENDING$.add(this);\n" + java.add &" synchronized ({className}.class) {{ if (released.compareAndSet(false, true)) {{ LIVE$.remove(this); if (generation == GENERATION$) PENDING$.add(this); }} }}\n" java.add " }\n" java.add " }\n\n" java.add " /** Loads the native library from an absolute path. */\n" @@ -1003,6 +1021,7 @@ proc writeJava*(dir, lib: string): NimNode = java.add " if (r.generation == GENERATION$) r.unref.accept(r.handle);\n" java.add " }\n" java.add " }\n\n" + java.add " private static void releaseGeneration$() { for (Release$ r : LIVE$) r.run(); releasePending(); }\n" java.add " /**\n" java.add " * Development only: binds the library to the Nim library at absolutePath. Needs a\n" java.add " * loader built with GENNY_JAVA_HOT_RELOAD, otherwise it throws UnsupportedOperationException.\n" @@ -1010,15 +1029,26 @@ proc writeJava*(dir, lib: string): NimNode = java.add " * A failed reload throws IllegalStateException and keeps the current library.\n" java.add " */\n" java.add " public static synchronized void reload(java.lang.String absolutePath) {\n" - java.add " PENDING$.clear();\n" - java.add " n_genny_reload(absolutePath);\n" - java.add " GENERATION$ = GENERATION$ + 1;\n" + java.add " prepareReload(absolutePath);\n" + java.add " activateReload();\n" + java.add " commitReload();\n" java.add " }\n\n" java.add " /** True when the loaded library was built with GENNY_JAVA_HOT_RELOAD. */\n" java.add " public static boolean hotReloadable() {\n" java.add " return n_genny_hot_reloadable();\n" java.add " }\n\n" java.add " static native void n_genny_reload(java.lang.String path);\n" + java.add &" public static final String ABI_FINGERPRINT = \"{abi}\";\n" + java.add " public static synchronized void prepareReload(String path) { releasePending(); n_genny_prepare(path); PREPARED$ = true; }\n" + java.add " public static synchronized void activateReload() { if (!PREPARED$ || ACTIVE$) throw new IllegalStateException(\"no prepared reload\"); releaseGeneration$(); n_genny_activate(); GENERATION$++; PREPARED$ = false; ACTIVE$ = true; }\n" + java.add " public static synchronized void rollbackReload() { if (!ACTIVE$) throw new IllegalStateException(\"no activated reload\"); releaseGeneration$(); n_genny_rollback(); GENERATION$++; ACTIVE$ = false; }\n" + java.add " public static synchronized void commitReload() { releasePending(); n_genny_commit(); ACTIVE$ = false; }\n" + java.add " public static synchronized void cancelReload() { n_genny_cancel(); PREPARED$ = false; }\n" + java.add " static native void n_genny_prepare(String path);\n" + java.add " static native void n_genny_activate();\n" + java.add " static native void n_genny_rollback();\n" + java.add " static native void n_genny_commit();\n" + java.add " static native void n_genny_cancel();\n" java.add " static native boolean n_genny_hot_reloadable();\n" java.add libMembers java.add "}\n" @@ -1086,6 +1116,9 @@ proc writeJava*(dir, lib: string): NimNode = code.add "#include \n" code.add "#include \n" code.add "#include \n" + code.add "#ifndef GENNY_JAVA_HOT_RELOAD\n" + code.add &"__attribute__((visibility(\"default\"))) const char *genny_abi_fingerprint(void) {{ return \"{abi}\"; }}\n" + code.add "#endif\n" code.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" code.add "#include \n" code.add "#include \n\n" @@ -1102,6 +1135,8 @@ proc writeJava*(dir, lib: string): NimNode = ("$lib_check_error", ("bool", "void")), ("$lib_take_error", ("char *", "void")) ]) + if needsStrings: + code.add protos(@[("$lib_java_release_string", ("void", "void"))]) code.add "static jclass genny_exception_class;\n" code.add "static jmethodID genny_exception_init;\n" for name in enumEntries.keys: @@ -1128,6 +1163,7 @@ proc writeJava*(dir, lib: string): NimNode = code.add "static void genny_raise(JNIEnv *env) {\n" code.add " char *message = $lib_take_error();\n" code.add " jstring jmessage = genny_jstring_from_utf8(env, message == NULL ? \"error\" : message);\n" + code.add " $lib_java_release_string();\n" code.add " jthrowable e;\n" code.add " if (jmessage == NULL) return;\n" code.add " e = (jthrowable)(*env)->NewObject(env, genny_exception_class, genny_exception_init, jmessage);\n" @@ -1145,62 +1181,99 @@ proc writeJava*(dir, lib: string): NimNode = code.add cHelpers code.add cWrappers - code.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" - code.add "/* Resolves every export in h; the pointers change only when all are found. */\n" - code.add "static const char *genny_bind(void *h) {\n" - if bound.len == 0: - code.add " (void)h;\n" - else: - code.add " static const char *const names[] = {\n" - for (name, _, _) in bound: - code.add &" \"{name}\",\n" - code.add " };\n" - code.add &" void *syms[{bound.len}];\n" - code.add " size_t i;\n" - code.add " for (i = 0; i < sizeof(names) / sizeof(names[0]); i++) {\n" - code.add " syms[i] = dlsym(h, names[i]);\n" - code.add " if (syms[i] == NULL) return names[i];\n" - code.add " }\n" - for i, (name, ret, params) in bound: - code.add &" {name} = ({ret} (*)({params}))syms[{i}];\n" - code.add " return NULL;\n" - code.add "}\n\n" - code.add "static void JNICALL genny_jni_genny_reload(JNIEnv *env, jclass cls, jstring path) {\n" - code.add " const char *p;\n" - code.add " const char *missing;\n" - code.add " void *h;\n" - code.add " (void)cls;\n" - code.add " if (path == NULL) {\n" - code.add " genny_throw(env, \"java/lang/NullPointerException\", \"path is null\");\n" - code.add " return;\n" - code.add " }\n" - code.add " p = (*env)->GetStringUTFChars(env, path, NULL);\n" - code.add " if (p == NULL) return;\n" - code.add " h = dlopen(p, RTLD_NOW | RTLD_LOCAL);\n" - code.add " (*env)->ReleaseStringUTFChars(env, path, p);\n" - code.add " if (h == NULL) {\n" - code.add " const char *e = dlerror();\n" - code.add " genny_throw(env, \"java/lang/IllegalStateException\", e == NULL ? \"dlopen failed\" : e);\n" - code.add " return;\n" - code.add " }\n" - code.add " missing = genny_bind(h);\n" - code.add " if (missing != NULL) {\n" - code.add " char message[512];\n" - code.add " snprintf(message, sizeof(message), \"native library is missing symbol %s\", missing);\n" - code.add " dlclose(h);\n" - code.add " genny_throw(env, \"java/lang/IllegalStateException\", message);\n" - code.add " return;\n" - code.add " }\n" - code.add " if (genny_handle != NULL) dlclose(genny_handle);\n" - code.add " genny_handle = h;\n" - code.add " genny_bound = 1;\n" - code.add "}\n" - code.add "#else\n" - code.add "static void JNICALL genny_jni_genny_reload(JNIEnv *env, jclass cls, jstring path) {\n" - code.add " (void)cls; (void)path;\n" - code.add " genny_throw(env, \"java/lang/UnsupportedOperationException\", \"built without GENNY_JAVA_HOT_RELOAD\");\n" - code.add "}\n" - code.add "#endif\n\n" + var names, assignments: string + for i, (name, ret, params) in bound: + names.add &" \"{name}\",\n" + assignments.add &" {name} = ({ret} (*)({params}))syms[{i}];\n" + code.add """ +#ifdef GENNY_JAVA_HOT_RELOAD +static void *genny_prepared, *genny_previous; +static int genny_activated; +static void *genny_active_syms[@COUNT@], *genny_prepared_syms[@COUNT@], *genny_previous_syms[@COUNT@]; +static const char *const genny_names[] = { +@NAMES@ +}; +static void genny_apply(void **syms) { +@ASSIGN@ +} +static void JNICALL genny_jni_genny_prepare(JNIEnv *env, jclass cls, jstring path) { + (void)cls; + if (genny_prepared || genny_activated) { + genny_throw(env, "java/lang/IllegalStateException", "reload transaction already active"); return; + } + if (!path) { genny_throw(env, "java/lang/NullPointerException", "path is null"); return; } + const char *p = (*env)->GetStringUTFChars(env, path, NULL); + if (!p) return; + void *h = dlopen(p, RTLD_NOW | RTLD_LOCAL); + (*env)->ReleaseStringUTFChars(env, path, p); + if (!h) { genny_throw(env, "java/lang/IllegalStateException", dlerror()); return; } + const char *(*abi)(void) = (const char *(*)(void))dlsym(h, "genny_abi_fingerprint"); + if (!abi || strcmp(abi(), "@ABI@")) { + dlclose(h); genny_throw(env, "java/lang/IllegalStateException", "native ABI fingerprint mismatch; restart required"); return; + } + for (size_t i = 0; i < @BOUND@; i++) { + genny_prepared_syms[i] = dlsym(h, genny_names[i]); + if (!genny_prepared_syms[i]) { + char message[512]; + snprintf(message, sizeof(message), "native library is missing symbol %s", genny_names[i]); + dlclose(h); genny_throw(env, "java/lang/IllegalStateException", message); return; + } + } + genny_prepared = h; +} +static void JNICALL genny_jni_genny_activate(JNIEnv *env, jclass cls) { + (void)cls; + if (!genny_prepared || genny_activated) { + genny_throw(env, "java/lang/IllegalStateException", "no prepared reload"); return; + } + genny_previous = genny_handle; + memcpy(genny_previous_syms, genny_active_syms, sizeof(genny_active_syms)); + genny_handle = genny_prepared; genny_prepared = NULL; + memcpy(genny_active_syms, genny_prepared_syms, sizeof(genny_active_syms)); + genny_apply(genny_active_syms); genny_bound = 1; genny_activated = 1; +} +static void JNICALL genny_jni_genny_commit(JNIEnv *env, jclass cls) { + (void)cls; + if (!genny_activated) { genny_throw(env, "java/lang/IllegalStateException", "no activated reload"); return; } + if (genny_previous) dlclose(genny_previous); + genny_previous = NULL; genny_activated = 0; +} +static void JNICALL genny_jni_genny_rollback(JNIEnv *env, jclass cls) { + (void)cls; + if (!genny_activated || !genny_previous) { + genny_throw(env, "java/lang/IllegalStateException", "no rollback generation"); return; + } + void *failed = genny_handle; + genny_handle = genny_previous; genny_previous = NULL; + memcpy(genny_active_syms, genny_previous_syms, sizeof(genny_active_syms)); + genny_apply(genny_active_syms); genny_activated = 0; + dlclose(failed); +} +static void JNICALL genny_jni_genny_cancel(JNIEnv *env, jclass cls) { + (void)cls; + if (genny_activated) { genny_throw(env, "java/lang/IllegalStateException", "activated reload requires rollback"); return; } + if (genny_prepared) dlclose(genny_prepared); + genny_prepared = NULL; +} +static void JNICALL genny_jni_genny_reload(JNIEnv *env, jclass cls, jstring path) { + genny_jni_genny_prepare(env, cls, path); + if ((*env)->ExceptionCheck(env)) return; + genny_jni_genny_activate(env, cls); + if ((*env)->ExceptionCheck(env)) return; + genny_jni_genny_commit(env, cls); +} +#else +#define GENNY_UNSUPPORTED() genny_throw(env, "java/lang/UnsupportedOperationException", "built without GENNY_JAVA_HOT_RELOAD") +static void JNICALL genny_jni_genny_prepare(JNIEnv *env, jclass cls, jstring path) { (void)cls; (void)path; GENNY_UNSUPPORTED(); } +static void JNICALL genny_jni_genny_reload(JNIEnv *env, jclass cls, jstring path) { (void)cls; (void)path; GENNY_UNSUPPORTED(); } +static void JNICALL genny_jni_genny_activate(JNIEnv *env, jclass cls) { (void)cls; GENNY_UNSUPPORTED(); } +static void JNICALL genny_jni_genny_rollback(JNIEnv *env, jclass cls) { (void)cls; GENNY_UNSUPPORTED(); } +static void JNICALL genny_jni_genny_commit(JNIEnv *env, jclass cls) { (void)cls; GENNY_UNSUPPORTED(); } +static void JNICALL genny_jni_genny_cancel(JNIEnv *env, jclass cls) { (void)cls; GENNY_UNSUPPORTED(); } +#endif + +""".replace("@COUNT@", $max(1, bound.len)).replace("@BOUND@", $bound.len) + .replace("@NAMES@", names).replace("@ASSIGN@", assignments).replace("@ABI@", abi) code.add "static jboolean JNICALL genny_jni_genny_hot_reloadable(JNIEnv *env, jclass cls) {\n" code.add " (void)env; (void)cls;\n" code.add "#ifdef GENNY_JAVA_HOT_RELOAD\n" @@ -1213,6 +1286,9 @@ proc writeJava*(dir, lib: string): NimNode = code.add "static JNINativeMethod genny_jni_methods[] = {\n" code.add cMethods code.add " {\"n_genny_reload\", \"(Ljava/lang/String;)V\", (void *)genny_jni_genny_reload},\n" + code.add " {\"n_genny_prepare\", \"(Ljava/lang/String;)V\", (void *)genny_jni_genny_prepare},\n" + for phase in ["activate", "rollback", "commit", "cancel"]: + code.add &" {{\"n_genny_{phase}\", \"()V\", (void *)genny_jni_genny_{phase}}},\n" code.add " {\"n_genny_hot_reloadable\", \"()Z\", (void *)genny_jni_genny_hot_reloadable},\n" code.add "};\n\n" code.add "JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) {\n" diff --git a/src/genny/languages/nim.nim b/src/genny/languages/nim.nim index 4e45756..71a0860 100644 --- a/src/genny/languages/nim.nim +++ b/src/genny/languages/nim.nim @@ -64,7 +64,10 @@ proc convertExportFromNim*(sym: NimNode): string = return "" else: if typ.repr == "string": - return ".cstring" + when defined(gennyJava): + return ".gennyJavaCString()" + else: + return ".cstring" elif typ.repr == "Rune": return ".int32" else: diff --git a/tests/TestJavaReload.java b/tests/TestJavaReload.java index 6a6bedb..e32b3ce 100644 --- a/tests/TestJavaReload.java +++ b/tests/TestJavaReload.java @@ -66,13 +66,32 @@ public static void main(String[] args) throws Exception { t = throwsType(IllegalStateException.class, () -> TestReload.reload(abs(args[3])), "library missing a symbol is rejected"); - check(t.getMessage().contains("test_reload_bonus"), "missing symbol is named"); + check(t.getMessage().contains("fingerprint"), "changed exported API is rejected before binding"); check(TestReload.version() == 2 && TestReload.bonus() == 7, "rejected library keeps the old one"); try (Box after = new Box(3)) { check(after.getValue() == 3, "handles still work after a failed reload"); } throwsType(NullPointerException.class, () -> TestReload.reload(null), "null path"); + Box survivor = new Box(17); + TestReload.prepareReload(abs(args[1])); + check(TestReload.version() == 2 && survivor.getValue() == 17, "prepare preserves active generation and handles"); + TestReload.cancelReload(); + check(survivor.getValue() == 17, "cancel preserves active handles"); + TestReload.prepareReload(abs(args[1])); + TestReload.activateReload(); + check(TestReload.version() == 1, "activation retains previous generation"); + throwsType(IllegalStateException.class, () -> survivor.getValue(), "activation invalidates old handles"); + Box candidate = new Box(19); + TestReload.rollbackReload(); + check(TestReload.version() == 2, "rollback restores previous function table"); + throwsType(IllegalStateException.class, () -> survivor.getValue(), "rollback never revives old handles"); + throwsType(IllegalStateException.class, () -> candidate.getValue(), "rollback invalidates candidate handles"); + survivor.close(); candidate.close(); + try (Box current = new Box(23)) { + throwsType(IllegalStateException.class, TestReload::activateReload, "unprepared activation rejected"); + check(current.getValue() == 23, "unprepared activation preserves handles"); + } System.out.println("All Java reload tests passed"); } }