[WIP] Java jni

.github/workflows/java.yml

@@ -0,0 +1,48 @@
+name: Java
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: treeform/setup-nim-action@v6
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: '17'
+    - name: Build standard test DLL with the Java backend
+      run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated/libtest.so tests/test.nim
+    - name: Run Java test against the standard test DLL
+      shell: bash
+      run: |
+        javac -d tests/generated/classes tests/generated/java/genny/*.java tests/TestJavaStandard.java
+        java -Xcheck:jni -cp tests/generated/classes TestJavaStandard tests/generated/libtest.so 2>&1 | tee java_standard.log
+        ! grep -n WARNING java_standard.log
+    - name: Build Java test DLL
+      run: nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:tests/generated_java/libtest_java.so tests/test_java.nim
+    - name: Compile Java test
+      run: javac -d tests/generated_java/classes tests/generated_java/java/genny/*.java tests/TestJavaMain.java
+    - name: Run Java test
+      shell: bash
+      run: |
+        java -Xcheck:jni -cp tests/generated_java/classes TestJavaMain tests/generated_java/libtest_java.so 2>&1 | tee java_main.log
+        ! grep -n WARNING java_main.log
+    - name: Build hot reload test libraries and loader
+      shell: bash
+      run: |
+        D=tests/generated_java_reload
+        nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:$D/libreload_v1.so tests/test_java_reload.nim
+        nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -d:reloadVersion=2 -o:$D/libreload_v2.so tests/test_java_reload.nim
+        nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -d:reloadDropSymbol -o:$D/libreload_drop.so tests/test_java_reload.nim
+        cc -shared -fPIC -DGENNY_JAVA_HOT_RELOAD -I$JAVA_HOME/include -I$JAVA_HOME/include/linux $D/test_reload_jni.c -ldl -o $D/libreload_loader.so
+        javac -d $D/classes $D/java/genny/*.java tests/TestJavaReload.java
+    - name: Run Java hot reload test
+      shell: bash
+      run: |
+        D=tests/generated_java_reload
+        java -Xcheck:jni -cp $D/classes TestJavaReload $D/libreload_loader.so $D/libreload_v1.so $D/libreload_v2.so $D/libreload_drop.so 2>&1 | tee java_reload.log
+        ! grep -n WARNING java_reload.log

.gitignore

@@ -17,3 +17,9 @@ tests/generated/*_native.c
 .*
 !.github/
 !.github/**
+tests/generated/*_jni.c
+tests/generated/java/
+tests/generated_java/
+*.class
+tests/generated_java_reload/
+tests/generated_java_reload_drop/

README.md

@@ -24,6 +24,69 @@ Nim           | {.importc.}   | ✅     | ✅     | ✅          | ✅    | ✅
 Python        | ctypes        | ✅     | ✅     | ✅          | ✅    | ✅ |
 Node.js       | ffi-napi      | ✅     | ✅     | ✅          | ✅    | no |
 C             | .h            | ✅     | ✅     | ✅          | ✅    | no |
+Java          | JNI           | ✅     | ✅     | ✅          | ✅    | ✅ |
+
+## Java
+
+Build the library with `JAVA_HOME` pointing at a JDK (9 or newer):
+
+```sh
+nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:libpixie.so bindings.nim
+javac -d classes bindings/generated/java/genny/*.java
+```
+
+`-d:noSignalHandler` is required: the JVM uses `SIGSEGV` internally. The package defaults to `genny`; change it with `-d:gennyJavaPackage=com.example.pixie`. Load the library once with `Pixie.load("/absolute/path/libpixie.so")`.
+
+* Ref objects and seqs are `AutoCloseable`. `close()` releases the Nim object right away; otherwise a `Cleaner` queues the release and it runs at the start of the next call into the library (or `Pixie.releasePending()`).
+* Nim's reference counts and the error state of raising procs are not thread-safe: call the library from one thread at a time.
+* Unsigned integers keep their bits in the signed Java type of the same size; Nim `int` is a Java `long`.
+* Strings are converted to and from UTF-8. A string containing `\0` is cut there, like in the C API.
+* `pointer` parameters take direct `ByteBuffer`s (or `null`); Nim gets the buffer's base address in native byte order and must not keep it after the call. Heap buffers throw `IllegalArgumentException`.
+* Value objects are copied in and out as C structs. Objects over 24 bytes must be `{.bycopy.}`; inherited, packed and variant objects are skipped with a warning.
+* Overloads that end up with the same Java signature get the overload prefix appended, e.g. `scaledInt64`.
+* Like the other backends, a ref object built by its exported constructor keeps one extra Nim reference and is never freed.
+
+### Hot reload (development)
+
+The JVM cannot unload a JNI library. For development, build the generated C on its own as a small loader with `-DGENNY_JAVA_HOT_RELOAD`; Java loads the loader once and it `dlopen`s the real Nim library, so the Nim code can be rebuilt and swapped while the JVM keeps running. It uses `dlopen`, so it is not available on Windows:
+
+```sh
+nim c --mm:arc --app:lib -d:gennyJava -d:noSignalHandler -o:build/libpixie-1.so bindings.nim
+cc -shared -fPIC -DGENNY_JAVA_HOT_RELOAD -I$JAVA_HOME/include -I$JAVA_HOME/include/linux \
+  bindings/generated/pixie_jni.c -ldl -o libpixie_loader.so
+```
+
+The generated class provides `prepareReload(path)`, `activateReload()`,
+`rollbackReload()`, `commitReload()` and `cancelReload()`. Preparation checks
+every symbol and the generated ABI fingerprint without changing the active
+table. Activation retains the previous table and library handle; commit
+unloads the previous library, and rollback restores it after the failed
+candidate has been disposed. The caller must quiesce all native calls and
+stop every native thread before retiring a generation. Run untrusted candidate
+initialization in a separate process before calling prepare.
+
+`ABI_FINGERPRINT` covers exported signatures, value layouts, ref-object
+accessors, enums, constants and the error protocol. Incompatible API changes
+are rejected and need new Java bindings. Every activation and rollback
+advances the Java handle generation; rollback never revives old handles.
+Live and queued native references are released before changing generations.
+Nim string results remain owned by a thread-local holder until JNI finishes
+copying them, then JNI releases the holder. This includes error messages and
+string getters; returned pointers never outlive their Nim storage.
+`reload(path)` remains a convenience prepare/activate/commit sequence for
+applications that have already stopped their native runtime.
+
+```java
+Pixie.load("/abs/libpixie_loader.so");
+Pixie.reload("/abs/build/libpixie-1.so"); // and again after every rebuild
+```
+
+* Every call throws `IllegalStateException` until the first successful `reload`. `Pixie.hotReloadable()` tells a loader from a normal build, where `reload` throws `UnsupportedOperationException`.
+* A reload that cannot open the library or finds a missing export throws `IllegalStateException` and keeps the current library.
+* Each activation and rollback starts a new generation. Ref objects and seqs from an older generation throw `IllegalStateException` when used. Their references are released before switching; closing or collecting stale handles does nothing.
+* Copy each build to a new path before reloading: `dlopen` hands back the already loaded library for the same file.
+* Stop every thread that runs Nim code, and every call into the library, before reloading: the old library is unloaded.
+* The exported API must stay the same between reloads; regenerate and rebuild the loader (and restart the JVM) when it changes.
 
 ## Quest for "nice" language binding.
 

src/genny.nim

@@ -7,9 +7,10 @@ when defined(gennyNode): import genny/languages/node
 when defined(gennyPython): import genny/languages/python
 when defined(gennyPythonNative): import genny/languages/python_native
 when defined(gennyZig): import genny/languages/zig
+when defined(gennyJava): import genny/languages/java
 
-when not defined(gennyNim) and not defined(gennyPython) and not defined(gennyPythonNative) and not defined(gennyNode) and not defined(gennyC) and not defined(gennyCpp) and not defined(gennyZig):
-  {.error: "Please define one of the genny languages. Use -d:gennyNim, -d:gennyPython, -d:gennyPythonNative, -d:gennyNode, -d:gennyC, -d:gennyCpp, -d:gennyZig to define the languages you want to export.".}
+when not defined(gennyNim) and not defined(gennyPython) and not defined(gennyPythonNative) and not defined(gennyNode) and not defined(gennyC) and not defined(gennyCpp) and not defined(gennyZig) and not defined(gennyJava):
+  {.error: "Please define one of the genny languages. Use -d:gennyNim, -d:gennyPython, -d:gennyPythonNative, -d:gennyNode, -d:gennyC, -d:gennyCpp, -d:gennyZig, -d:gennyJava to define the languages you want to export.".}
 
 template discard2(f: untyped): untyped =
   when(compiles do: discard f):
@@ -49,6 +50,7 @@ macro exportConstsTyped(body: typed) =
     when defined(gennyC): exportConstC(sym)
     when defined(gennyCpp): exportConstCpp(sym)
     when defined(gennyZig): exportConstZig(sym)
+    when defined(gennyJava): exportConstJava(sym)
 
 template exportConsts*(body: untyped) =
   ## Exports a list of constants.
@@ -74,6 +76,7 @@ macro exportEnumsTyped(body: typed) =
     when defined(gennyC): exportEnumC(sym)
     when defined(gennyCpp): exportEnumCpp(sym)
     when defined(gennyZig): exportEnumZig(sym)
+    when defined(gennyJava): exportEnumJava(sym)
 
 template exportEnums*(body: untyped) =
   ## Exports a list of enums.
@@ -133,6 +136,7 @@ proc procTyped(
   when defined(gennyC): exportProcC(procSym, owner, prefixes)
   when defined(gennyCpp): exportProcCpp(procSym, owner, prefixes)
   when defined(gennyZig): exportProcZig(procSym, owner, prefixes)
+  when defined(gennyJava): exportProcJava(procSym, owner, prefixes)
 
 macro exportProcsUntyped(body: untyped) =
   result = newNimNode(nnkStmtList)
@@ -195,6 +199,7 @@ macro exportObjectTyped(body: typed) =
   when defined(gennyC): exportObjectC(sym, constructor)
   when defined(gennyCpp): exportObjectCpp(sym, constructor)
   when defined(gennyZig): exportObjectZig(sym, constructor)
+  when defined(gennyJava): exportObjectJava(sym, constructor)
 
   if procsBlock[1].len > 0:
     var procsSeen: seq[string]
@@ -216,6 +221,7 @@ macro exportObjectTyped(body: typed) =
       when defined(gennyC): exportProcC(procSym, sym, prefixes)
       when defined(gennyCpp): exportProcCpp(procSym, sym, prefixes)
       when defined(gennyZig): exportProcZig(procSym, sym, prefixes)
+      when defined(gennyJava): exportProcJava(procSym, sym, prefixes)
 
   when defined(gennyZig): exportCloseObjectZig()
   when defined(gennyCpp): exportCloseObjectCpp()
@@ -256,6 +262,7 @@ macro exportSeqTyped(body: typed) =
   when defined(gennyC): exportSeqC(sym)
   when defined(gennyCpp): exportSeqCpp(sym)
   when defined(gennyZig): exportSeqZig(sym)
+  when defined(gennyJava): exportSeqJava(sym)
 
   for entry in body.asStmtList()[1 .. ^1]:
     procTyped(entry, sym)
@@ -336,6 +343,7 @@ macro exportRefObjectTyped(body: typed) =
   when defined(gennyC): exportRefObjectC(sym, fields, constructor)
   when defined(gennyCpp): exportRefObjectCpp(sym, fields, constructor)
   when defined(gennyZig): exportRefObjectZig(sym, fields, constructor)
+  when defined(gennyJava): exportRefObjectJava(sym, fields, constructor)
 
   if procsBlock[1].len > 0:
     var procsSeen: seq[string]
@@ -357,6 +365,7 @@ macro exportRefObjectTyped(body: typed) =
       when defined(gennyC): exportProcC(procSym, sym, prefixes)
       when defined(gennyCpp): exportProcCpp(procSym, sym, prefixes)
       when defined(gennyZig): exportProcZig(procSym, sym, prefixes)
+      when defined(gennyJava): exportProcJava(procSym, sym, prefixes)
 
   when defined(gennyCpp): exportCloseObjectCpp()
   when defined(gennyZig): exportCloseObjectZig()
@@ -380,3 +389,4 @@ macro writeFiles*(dir, lib: static[string]) =
   when defined(gennyC): writeC(dir, lib)
   when defined(gennyCpp): writeCpp(dir, lib)
   when defined(gennyZig): writeZig(dir, lib)
+  when defined(gennyJava): result.add writeJava(dir, lib)

src/genny/internal.nim

@@ -265,7 +265,18 @@ when (NimMajor, NimMinor, NimPatch) == (1, 6, 2):
 
 proc writeInternal*(dir, lib: string) =
   createDir(dir)
+  let stringHeader = when defined(gennyJava): """
+var gennyJavaReturn {.threadvar.}: string
+
+proc gennyJavaCString(value: string): cstring =
+  gennyJavaReturn = value
+  gennyJavaReturn.cstring
+
+proc $lib_java_release_string*() {.raises: [], cdecl, exportc, dynlib.} =
+  gennyJavaReturn = ""
+
+""" else: ""
   writeFile(
     &"{dir}/internal.nim",
-    header & internal.replace("$Lib", lib).replace("$lib", toSnakeCase(lib))
+    (header & stringHeader & internal).replace("$Lib", lib).replace("$lib", toSnakeCase(lib))
   )