diff --git a/samples/cs/Shared/Utils.cs b/samples/cs/Shared/Utils.cs index 316248f5f..b300b6abf 100644 --- a/samples/cs/Shared/Utils.cs +++ b/samples/cs/Shared/Utils.cs @@ -53,6 +53,12 @@ internal static async Task RunWithSpinner(string msg, T workTask, bool warnOn private static async Task ShowSpinner(string msg, CancellationToken token) { + if (Console.IsOutputRedirected) + { + Console.WriteLine($"{msg}..."); + return; + } + Console.OutputEncoding = Encoding.UTF8; var sequence = new[] { '◴','◷','◶','◵' }; diff --git a/samples/cs/live-audio-transcription/Program.cs b/samples/cs/live-audio-transcription/Program.cs index 9caa2569d..6e1113149 100644 --- a/samples/cs/live-audio-transcription/Program.cs +++ b/samples/cs/live-audio-transcription/Program.cs @@ -171,5 +171,6 @@ await model.DownloadAsync(progress => await session.StopAsync(); await readTask; +await session.DisposeAsync(); await model.UnloadAsync(); diff --git a/samples/cs/tool-calling-foundry-local-sdk/Program.cs b/samples/cs/tool-calling-foundry-local-sdk/Program.cs index a3b38b08f..39eab0cdd 100644 --- a/samples/cs/tool-calling-foundry-local-sdk/Program.cs +++ b/samples/cs/tool-calling-foundry-local-sdk/Program.cs @@ -3,6 +3,7 @@ using Microsoft.AI.Foundry.Local; using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels; using Betalgo.Ranul.OpenAI.ObjectModels.SharedModels; +using ToolChoice = Betalgo.Ranul.OpenAI.ObjectModels.RequestModels.ToolChoice; using System.Text.Json; // diff --git a/samples/cs/tutorial-tool-calling/Program.cs b/samples/cs/tutorial-tool-calling/Program.cs index 5ae60419a..111279e22 100644 --- a/samples/cs/tutorial-tool-calling/Program.cs +++ b/samples/cs/tutorial-tool-calling/Program.cs @@ -5,6 +5,7 @@ using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels; using Betalgo.Ranul.OpenAI.ObjectModels.ResponseModels; using Betalgo.Ranul.OpenAI.ObjectModels.SharedModels; +using ToolChoice = Betalgo.Ranul.OpenAI.ObjectModels.RequestModels.ToolChoice; using Microsoft.Extensions.Logging; // diff --git a/samples/js/live-audio-transcription/app.js b/samples/js/live-audio-transcription/app.js index e7672cdd2..4dc669cc1 100644 --- a/samples/js/live-audio-transcription/app.js +++ b/samples/js/live-audio-transcription/app.js @@ -184,6 +184,7 @@ try { await new Promise((resolve) => setTimeout(resolve, 3000)); await session.stop(); await readPromise; + await session.dispose(); await model.unload(); console.log('✓ Done'); process.exit(0); @@ -197,6 +198,7 @@ process.on('SIGINT', async () => { } await session.stop(); await readPromise; + await session.dispose(); await model.unload(); console.log('✓ Done'); process.exit(0); diff --git a/samples/python/live-audio-transcription/README.md b/samples/python/live-audio-transcription/README.md index dc8a263ad..4bcbfde95 100644 --- a/samples/python/live-audio-transcription/README.md +++ b/samples/python/live-audio-transcription/README.md @@ -65,4 +65,5 @@ for result in session.get_stream(): print(result.is_final) # True for final results session.stop() +session.close() ``` diff --git a/samples/python/live-audio-transcription/src/app.py b/samples/python/live-audio-transcription/src/app.py index 07bbc12c8..62ed81168 100644 --- a/samples/python/live-audio-transcription/src/app.py +++ b/samples/python/live-audio-transcription/src/app.py @@ -168,6 +168,7 @@ def shutdown(*_args): session.stop() read_thread.join(timeout=5) + session.close() model.unload() print("✓ Done") sys.exit(0) diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h index 161b9d56e..bca4655d2 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h @@ -792,6 +792,7 @@ class ICatalog { /// Get every individual model variant currently present in the local cache. virtual ModelList GetCachedModels() const = 0; + /// Get every loaded leaf variant, independently of each alias group's selected variant. virtual ModelList GetLoadedModels() const = 0; virtual std::unique_ptr GetModel(const std::string& alias) const = 0; virtual std::unique_ptr GetModelVariant(const std::string& model_id) const = 0; diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.cc b/sdk_v2/cpp/src/catalog/base_model_catalog.cc index 18c512651..e754b51c8 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.cc @@ -418,8 +418,14 @@ std::vector BaseModelCatalog::GetLoadedModels() const { std::lock_guard lock(mutex_); std::vector result; for (auto& stored : models_) { - if (stored.active && stored.model->IsLoaded()) { - result.push_back(stored.model.get()); + if (!stored.active) { + continue; + } + + for (auto* variant : stored.model->Variants()) { + if (variant->IsLoaded()) { + result.push_back(variant); + } } } diff --git a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc index 4da5c5f7c..ae691c741 100644 --- a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc @@ -10,6 +10,7 @@ // #include "catalog/base_model_catalog.h" #include "internal_api/test_helpers.h" +#include "internal_api/test_model_cache.h" #include "logger.h" #include "model.h" #include "model_info.h" @@ -140,6 +141,55 @@ TEST_F(BaseModelCatalogTest, GetName_ReturnsNameFromConstruction) { EXPECT_EQ(catalog.GetName(), "test-catalog"); } +TEST_F(BaseModelCatalogTest, GetLoadedModelsReturnsNonSelectedLeafVariants) { + fl::test::FakeServiceBindings bindings; + TestCatalog catalog(logger_); + const auto path = fl::test::GetTestDataPath("tiny-random-gpt2-fp32-1").string(); + for (const int version : {2, 1}) { + ModelInfo info; + info.model_id = "loaded-family:" + std::to_string(version); + info.name = "loaded-family"; + info.version = version; + info.alias = "loaded-family"; + catalog.AddModel(Model::FromModelInfo(std::move(info), path, bindings.download_manager, + bindings.model_load_manager)); + } + + auto* group = catalog.GetModel("loaded-family"); + auto* loaded = catalog.GetModelVariant("loaded-family:1"); + ASSERT_NE(group, nullptr); + ASSERT_NE(loaded, nullptr); + ASSERT_EQ(group->Info().model_id, "loaded-family:2"); + EXPECT_TRUE(catalog.GetLoadedModels().empty()); + + loaded->Load(ExecutionProvider::kCPU); + ASSERT_TRUE(loaded->IsLoaded()); + ASSERT_FALSE(group->IsLoaded()); + + auto listed = catalog.GetLoadedModels(); + ASSERT_EQ(listed.size(), 1u); + EXPECT_EQ(listed.front(), loaded); + EXPECT_EQ(listed.front()->Variants().size(), 1u); + EXPECT_EQ(group->Info().model_id, "loaded-family:2"); + + auto* selected = catalog.GetModelVariant("loaded-family:2"); + ASSERT_NE(selected, nullptr); + selected->Load(ExecutionProvider::kCPU); + listed = catalog.GetLoadedModels(); + ASSERT_EQ(listed.size(), 2u); + EXPECT_EQ(listed[0], selected); + EXPECT_EQ(listed[1], loaded); + selected->Unload(); + + group->SelectVariant(*loaded); + listed = catalog.GetLoadedModels(); + ASSERT_EQ(listed.size(), 1u); + EXPECT_EQ(listed.front(), loaded); + + loaded->Unload(); + EXPECT_TRUE(catalog.GetLoadedModels().empty()); +} + // ======================================================================== // GetModel // ======================================================================== diff --git a/sdk_v2/python/README.md b/sdk_v2/python/README.md index dbc348d86..d0f351f69 100644 --- a/sdk_v2/python/README.md +++ b/sdk_v2/python/README.md @@ -495,6 +495,10 @@ Common session methods: Enums: `ItemType`, `TextItemType`, `MessageRole`, `TensorDataType`. +Inline `BytesItem`, `ImageItem`, and `AudioItem` data is copied into item-owned storage. After transferring an item to a +`Request` or `ItemQueue`, the Python wrapper and original input buffer can be discarded; the bytes remain alive until the +native item is released. + ### CLI entry point | Function | CLI name | Description | diff --git a/sdk_v2/python/src/foundry_local_sdk/items.py b/sdk_v2/python/src/foundry_local_sdk/items.py index 8d2552743..c56af4a9a 100644 --- a/sdk_v2/python/src/foundry_local_sdk/items.py +++ b/sdk_v2/python/src/foundry_local_sdk/items.py @@ -21,6 +21,39 @@ # C float to. cffi cdef does not process #define. _CONFIDENCE_UNSET: float = -struct.unpack(" None: + from foundry_local_sdk._native import ffi + + del _native_buffers[int(ffi.cast("size_t", user_data))] + + +def _set_owned_buffer(item_ptr, native_data, raw: bytes, setter) -> None: + from foundry_local_sdk._native import ffi + from foundry_local_sdk._native.api import api + + # SetBytes/SetImage/SetAudio borrow data. Native ownership can outlive the Python wrapper after a queue/request transfer. + buffer = ffi.new("unsigned char[]", raw) + key = int(ffi.cast("size_t", buffer)) + callback_type = ffi.typeof(native_data.deleter) + if callback_type not in _buffer_deleters: + _buffer_deleters[callback_type] = ffi.callback(callback_type, _release_native_buffer) + + _native_buffers[key] = buffer + native_data.data = buffer + native_data.mutable_data = buffer + native_data.data_size = len(raw) + native_data.deleter = _buffer_deleters[callback_type] + native_data.deleter_user_data = buffer + try: + api.check_status(setter(item_ptr, native_data)) + except BaseException: + del _native_buffers[key] + raise + def _validate_native_string(value: str, argument_name: str) -> None: if not isinstance(value, str): @@ -389,15 +422,10 @@ def __init__(self, data: bytes | bytearray | memoryview) -> None: raw = bytes(data) self.data = raw - # Keep both raw and buf alive until after SetBytes returns — the native - # call reads the buffer pointer synchronously so no heap copy is needed. - buf = ffi.from_buffer(raw) bytes_data = ffi.new("flBytesData*") bytes_data.version = _API_VERSION bytes_data.item_type = int(ItemType.BYTES) - bytes_data.data = ffi.cast("void *", buf) - bytes_data.data_size = len(raw) - api.check_status(api.item.SetBytes(self._ptr, bytes_data)) + _set_owned_buffer(self._ptr, bytes_data, raw, api.item.SetBytes) @classmethod def _from_native(cls, ptr, owns: bool) -> "BytesItem": @@ -444,16 +472,12 @@ def __init__(self, format: str, data: bytes | bytearray) -> None: raw = bytes(data) self.data = raw - # Keep all cffi temporaries alive until after SetImage returns. - buf = ffi.from_buffer(raw) c_fmt = ffi.new("char[]", format.encode("utf-8") + b"\x00") image_data = ffi.new("flImageData*") image_data.version = _API_VERSION - image_data.data = ffi.cast("void *", buf) - image_data.data_size = len(raw) image_data.format = c_fmt image_data.uri = ffi.NULL - api.check_status(api.item.SetImage(self._ptr, image_data)) + _set_owned_buffer(self._ptr, image_data, raw, api.item.SetImage) @classmethod def from_uri(cls, uri: str, format: str | None = None) -> "ImageItem": @@ -542,18 +566,14 @@ def __init__( self.sample_rate = sample_rate self.channels = channels - # Keep all cffi temporaries alive until after SetAudio returns. - buf = ffi.from_buffer(raw) c_fmt = ffi.new("char[]", format.encode("utf-8") + b"\x00") audio_data = ffi.new("flAudioData*") audio_data.version = _API_VERSION - audio_data.data = ffi.cast("void *", buf) - audio_data.data_size = len(raw) audio_data.format = c_fmt audio_data.uri = ffi.NULL audio_data.sample_rate = sample_rate audio_data.channels = channels - api.check_status(api.item.SetAudio(self._ptr, audio_data)) + _set_owned_buffer(self._ptr, audio_data, raw, api.item.SetAudio) @classmethod def create_format_descriptor(cls, format: str, sample_rate: int, channels: int) -> "AudioItem": diff --git a/sdk_v2/python/test/unit/test_items.py b/sdk_v2/python/test/unit/test_items.py index 1d36728d5..f0c16307b 100644 --- a/sdk_v2/python/test/unit/test_items.py +++ b/sdk_v2/python/test/unit/test_items.py @@ -197,6 +197,59 @@ def test_repr_contains_class_name(self): assert "BytesItem" in repr(BytesItem(b"hi")) +class TestBufferOwnership: + @pytest.mark.parametrize("native_type", ["flBytesData", "flImageData", "flAudioData"]) + def test_failed_setter_releases_buffer(self, native_type): + from foundry_local_sdk._native import ffi + from foundry_local_sdk import items + + previous = set(items._native_buffers) + + def fail(_ptr, _data): + raise RuntimeError("setter failed") + + with pytest.raises(RuntimeError, match="setter failed"): + items._set_owned_buffer(ffi.NULL, ffi.new(f"{native_type}*"), b"payload", fail) + assert set(items._native_buffers) == previous + + @pytest.mark.parametrize("item_class", [BytesItem, ImageItem, AudioItem]) + @pytest.mark.parametrize("payload", [b"", bytes(i % 251 for i in range(3200))]) + def test_transferred_buffer_survives_wrapper_collection(self, item_class, payload): + from foundry_local_sdk.item_queue import ItemQueue + from foundry_local_sdk import items + + previous = set(items._native_buffers) + with ItemQueue() as queue: + raw = bytearray(payload) + item = item_class(raw) if item_class is BytesItem else item_class("pcm", raw) + queue.push(item) + del item, raw + gc.collect() + churn = [bytes(bytearray([i % 255] * 3200)) for i in range(500)] + with queue.try_pop() as readback: + assert isinstance(readback, item_class) + assert readback.data == payload + del churn + assert set(items._native_buffers) == previous + + @pytest.mark.parametrize("item_class", [BytesItem, ImageItem, AudioItem]) + def test_request_owns_buffer_until_native_release(self, item_class): + from foundry_local_sdk.request import Request + from foundry_local_sdk import items + + previous = set(items._native_buffers) + payload = bytearray(b"keep alive") + with Request() as request: + item = item_class(payload) if item_class is BytesItem else item_class("pcm", payload) + request.add_item(item) + del item + payload[:] = b"overwritten" + gc.collect() + assert request.get_item(0).data == b"keep alive" + assert set(items._native_buffers) != previous + assert set(items._native_buffers) == previous + + class TestImageItem: def test_inline_data_construct(self): item = ImageItem("png", b"\x89PNG\r\n")