diff --git a/CMakeLists.txt b/CMakeLists.txt index a1df03cd13..904c90bca3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -375,6 +375,7 @@ set(SIMPLNX_HDRS ${SIMPLNX_SOURCE_DIR}/Common/Constants.hpp ${SIMPLNX_SOURCE_DIR}/Common/DataTypeUtilities.hpp ${SIMPLNX_SOURCE_DIR}/Common/DataVector.hpp + ${SIMPLNX_SOURCE_DIR}/Common/BoundedRead.hpp ${SIMPLNX_SOURCE_DIR}/Common/Extent.hpp ${SIMPLNX_SOURCE_DIR}/Common/EulerAngle.hpp ${SIMPLNX_SOURCE_DIR}/Common/Filesystem.hpp @@ -868,6 +869,7 @@ set(SIMPLNX_SRCS ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/ChunkShapePolicy.cpp ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/DeflateEligibility.cpp ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/ParallelChunkCodec.cpp + ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/BoundedChunkRead.cpp ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/IO/DatasetIO.cpp ${SIMPLNX_SOURCE_DIR}/Utilities/Parsing/HDF5/IO/FileIO.cpp @@ -1021,6 +1023,30 @@ foreach(plugin ${SIMPLNX_PLUGIN_LIST}) simplnx_add_plugin(PLUGIN_NAME ${plugin}) endforeach() +# Register the completion acceptance cases after their real fixture/writer plugins exist. +if(BUILD_TESTING AND SIMPLNX_BUILD_TESTS AND SIMPLNX_PLUGIN_ENABLE_TestOne AND SIMPLNX_ENABLE_SimplnxCore AND SIMPLNX_PLUGIN_ENABLE_SimplnxCore) + foreach(_completion_target IN ITEMS nxrunner TestOne SimplnxCore) + if(NOT TARGET ${_completion_target}) + message(FATAL_ERROR "Completion CLI tests require target '${_completion_target}' after plugin registration") + endif() + endforeach() + add_dependencies(nxrunner TestOne SimplnxCore) + foreach(_completion_mode IN ITEMS unarmed flush execute_and_flush allocation snapshot root_snapshot snapshot_allocation) + add_test(NAME "nxrunner_pipeline_flush_${_completion_mode}" + COMMAND ${CMAKE_COMMAND} + "-DNXRUNNER_EXECUTABLE=$" + "-DTEST_MODE=${_completion_mode}" + "-DSCRATCH_ROOT=${simplnx_BINARY_DIR}/CTestPipelineFlush/${_completion_mode}/$" + -P "${simplnx_SOURCE_DIR}/src/nxrunner/test/PipelineFlushReachesUserTest.cmake" + ) + set_tests_properties("nxrunner_pipeline_flush_${_completion_mode}" PROPERTIES + RUN_SERIAL TRUE + TIMEOUT 120 + WORKING_DIRECTORY $ + ) + endforeach() +endif() + # Benchmarking utility if(SIMPLNX_ENABLE_BENCHMARK_UTILITY) add_subdirectory(utilities/benchmark) diff --git a/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshot.hpp b/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshot.hpp new file mode 100644 index 0000000000..b523281a02 --- /dev/null +++ b/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshot.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include "simplnx/Core/Preferences.hpp" + +#include + +#include + +namespace ip_bench +{ +/** + * @brief Restores exact explicit storage preferences after a benchmark scope. + * + * The benchmark tools do not link UnitTestCommon. This guard changes only the + * in-memory Preferences object and does not save a preferences file. + */ +class BenchmarkPreferenceSnapshot +{ +public: + /** + * @brief Validates and applies a temporary storage mode and threshold. + * @param preferences Preference object to change and restore. + * @param mode Temporary storage mode. + * @param temporaryThreshold Signed temporary threshold in bytes. + */ + BenchmarkPreferenceSnapshot(nx::core::Preferences& preferences, nx::core::DataStorageMode mode, nx::core::int64 temporaryThreshold = 0) + : m_Preferences(preferences) + , m_HadMode(preferences.contains(nx::core::Preferences::k_DataStorageMode_Key)) + , m_OriginalMode(preferences.value(nx::core::Preferences::k_DataStorageMode_Key)) + , m_HadThreshold(preferences.contains(nx::core::Preferences::k_LargeDataSize_Key)) + , m_OriginalThreshold(preferences.value(nx::core::Preferences::k_LargeDataSize_Key)) + { + const nlohmann::json proposed(temporaryThreshold); + const auto validation = nx::core::Preferences::ValidateOocSizeValue(nx::core::Preferences::k_LargeDataSize_Key, proposed); + if(validation.invalid()) + { + throw std::invalid_argument(validation.errors().front().message); + } + m_Preferences.setDataStorageMode(mode); + m_Preferences.setValue(nx::core::Preferences::k_LargeDataSize_Key, proposed); + } + + ~BenchmarkPreferenceSnapshot() + { + if(m_HadThreshold) + { + m_Preferences.setValue(nx::core::Preferences::k_LargeDataSize_Key, m_OriginalThreshold); + } + else + { + m_Preferences.removeValue(nx::core::Preferences::k_LargeDataSize_Key); + } + if(m_HadMode) + { + m_Preferences.setValue(nx::core::Preferences::k_DataStorageMode_Key, m_OriginalMode); + } + else + { + m_Preferences.removeValue(nx::core::Preferences::k_DataStorageMode_Key); + } + } + + BenchmarkPreferenceSnapshot(const BenchmarkPreferenceSnapshot&) = delete; + BenchmarkPreferenceSnapshot(BenchmarkPreferenceSnapshot&&) = delete; + BenchmarkPreferenceSnapshot& operator=(const BenchmarkPreferenceSnapshot&) = delete; + BenchmarkPreferenceSnapshot& operator=(BenchmarkPreferenceSnapshot&&) = delete; + +private: + nx::core::Preferences& m_Preferences; + bool m_HadMode = false; + nlohmann::json m_OriginalMode; + bool m_HadThreshold = false; + nlohmann::json m_OriginalThreshold; +}; +} // namespace ip_bench diff --git a/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshotTest.cpp b/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshotTest.cpp new file mode 100644 index 0000000000..f817074ee9 --- /dev/null +++ b/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshotTest.cpp @@ -0,0 +1,67 @@ +#define CATCH_CONFIG_MAIN + +#include "BenchmarkPreferenceSnapshot.hpp" + +#include + +#include +#include + +using namespace nx::core; + +TEST_CASE("Benchmark preference snapshot restores exact JSON and key presence", "[Core][Preferences][OocThreshold]") +{ + for(const auto& original : {nlohmann::json(int64{0}), nlohmann::json(int64{37}), nlohmann::json(uint64{0}), nlohmann::json(uint64{37}), nlohmann::json(std::numeric_limits::max())}) + { + DYNAMIC_SECTION(original.dump() << " type=" << static_cast(original.type())) + { + Preferences preferences; + preferences.setValue(Preferences::k_LargeDataSize_Key, original); + preferences.setDataStorageMode(DataStorageMode::Adaptive); + const auto mode = preferences.value(Preferences::k_DataStorageMode_Key); + { + ip_bench::BenchmarkPreferenceSnapshot snapshot(preferences, DataStorageMode::ForceInCore, 19); + CHECK(preferences.valueAs(Preferences::k_LargeDataSize_Key) == 19); + CHECK(preferences.dataStorageMode() == DataStorageMode::ForceInCore); + } + const auto restored = preferences.value(Preferences::k_LargeDataSize_Key); + REQUIRE(preferences.contains(Preferences::k_LargeDataSize_Key)); + REQUIRE(restored.type() == original.type()); + CHECK(restored.get() == original.get()); + CHECK(preferences.value(Preferences::k_DataStorageMode_Key) == mode); + CHECK(preferences.value(Preferences::k_DataStorageMode_Key).type() == mode.type()); + CHECK(preferences.contains(Preferences::k_DataStorageMode_Key)); + } + } + + Preferences absent; + absent.removeValue(Preferences::k_LargeDataSize_Key); + absent.removeValue(Preferences::k_DataStorageMode_Key); + { + ip_bench::BenchmarkPreferenceSnapshot snapshot(absent, DataStorageMode::ForceOutOfCore); + CHECK(absent.valueAs(Preferences::k_LargeDataSize_Key) == 0); + } + CHECK_FALSE(absent.contains(Preferences::k_LargeDataSize_Key)); + CHECK_FALSE(absent.contains(Preferences::k_DataStorageMode_Key)); +} + +TEST_CASE("Benchmark preference snapshot rejects a negative threshold before mode mutation", "[Core][Preferences][OocThreshold]") +{ + Preferences preferences; + preferences.setValue(Preferences::k_LargeDataSize_Key, std::numeric_limits::max()); + preferences.setDataStorageMode(DataStorageMode::Adaptive); + const auto originalThreshold = preferences.value(Preferences::k_LargeDataSize_Key); + const auto originalMode = preferences.value(Preferences::k_DataStorageMode_Key); + REQUIRE_THROWS_AS(ip_bench::BenchmarkPreferenceSnapshot(preferences, DataStorageMode::ForceOutOfCore, -1), std::invalid_argument); + CHECK(preferences.value(Preferences::k_LargeDataSize_Key) == originalThreshold); + CHECK(preferences.value(Preferences::k_LargeDataSize_Key).type() == originalThreshold.type()); + CHECK(preferences.value(Preferences::k_DataStorageMode_Key) == originalMode); + CHECK(preferences.value(Preferences::k_DataStorageMode_Key).type() == originalMode.type()); + CHECK(preferences.contains(Preferences::k_LargeDataSize_Key)); + CHECK(preferences.contains(Preferences::k_DataStorageMode_Key)); + + Preferences absent; + REQUIRE_THROWS_AS(ip_bench::BenchmarkPreferenceSnapshot(absent, DataStorageMode::ForceOutOfCore, -1), std::invalid_argument); + CHECK_FALSE(absent.contains(Preferences::k_LargeDataSize_Key)); + CHECK_FALSE(absent.contains(Preferences::k_DataStorageMode_Key)); +} diff --git a/src/Plugins/ImageProcessing/benchmark/BenchmarkRunner.cpp b/src/Plugins/ImageProcessing/benchmark/BenchmarkRunner.cpp index cf8e53e4ec..0f95d2bbc6 100644 --- a/src/Plugins/ImageProcessing/benchmark/BenchmarkRunner.cpp +++ b/src/Plugins/ImageProcessing/benchmark/BenchmarkRunner.cpp @@ -1,4 +1,5 @@ #include "BenchmarkRunner.hpp" +#include "BenchmarkPreferenceSnapshot.hpp" #include "BenchmarkTiling.hpp" @@ -38,37 +39,6 @@ std::string FirstErrorMessage(const std::vector& errors) return errors.empty() ? "(no message)" : errors.front().message; } -// RAII sentinel that mirrors UnitTest::PreferencesSentinel (save mode + large_data_size, apply, restore on -// destruction). Re-implemented locally because this standalone tool deliberately does NOT link the UnitTestCommon -// .cpp that defines PreferencesSentinel's ctor/dtor (see benchmark/CMakeLists.txt). This one restores in-memory only -// -- it never writes the preferences file to disk. -class BenchPreferencesSentinel -{ -public: - BenchPreferencesSentinel(DataStorageMode mode, int64 largeDataSize) - { - auto* prefs = Application::Instance()->getPreferences(); - m_OriginalMode = prefs->dataStorageMode(); - m_OriginalSize = prefs->valueAs(Preferences::k_LargeDataSize_Key); - prefs->setDataStorageMode(mode); - prefs->setValue(Preferences::k_LargeDataSize_Key, largeDataSize); - } - ~BenchPreferencesSentinel() - { - auto* prefs = Application::Instance()->getPreferences(); - prefs->setDataStorageMode(m_OriginalMode); - prefs->setValue(Preferences::k_LargeDataSize_Key, m_OriginalSize); - } - BenchPreferencesSentinel(const BenchPreferencesSentinel&) = delete; - BenchPreferencesSentinel(BenchPreferencesSentinel&&) = delete; - BenchPreferencesSentinel& operator=(const BenchPreferencesSentinel&) = delete; - BenchPreferencesSentinel& operator=(BenchPreferencesSentinel&&) = delete; - -private: - DataStorageMode m_OriginalMode; - int64 m_OriginalSize; -}; - // Removes every DataObject NOT present in @p baseline (the set of IDs snapshotted right after the tiled input was // built). Handles both in-place-output filters (an array under the shared cell AM) and new-geometry-output filters // like MaximumProjection (a whole new ImageGeom). removeData recurses into children and is a no-op for IDs already @@ -397,7 +367,7 @@ BenchmarkRow RunFilterBenchmark(const BenchmarkFilterSpec& spec, const SweepOpti DataStructure dsInCore; DataPath inputPath; { - const BenchPreferencesSentinel sentinel(DataStorageMode::ForceInCore, 0); + const ip_bench::BenchmarkPreferenceSnapshot sentinel(*Application::Instance()->getPreferences(), DataStorageMode::ForceInCore); Result r = PrepareBenchmarkInput(dsInCore, spec, geom, cellAmName, inArrayName, opts.targetVoxels, opts.minZSlices, opts.force2D, DataStorageMode::ForceInCore); if(r.invalid()) { @@ -418,7 +388,7 @@ BenchmarkRow RunFilterBenchmark(const BenchmarkFilterSpec& spec, const SweepOpti // --- new-in-core --- if(!opts.oocOnly) { - const BenchPreferencesSentinel sentinel(DataStorageMode::ForceInCore, 0); + const ip_bench::BenchmarkPreferenceSnapshot sentinel(*Application::Instance()->getPreferences(), DataStorageMode::ForceInCore); std::string note; row.newIncoreMs = RunConfig(filterList, spec.newFilterUuid, dsInCore, baseInCore, spec, geom, inputPath, outName, opts.warmup, opts.repeats, opts.projectionAxis, note); if(!note.empty()) @@ -435,7 +405,7 @@ BenchmarkRow RunFilterBenchmark(const BenchmarkFilterSpec& spec, const SweepOpti DataPath oocInput; bool built = false; { - const BenchPreferencesSentinel sentinel(DataStorageMode::ForceOutOfCore, opts.oocThreshold); + const ip_bench::BenchmarkPreferenceSnapshot sentinel(*Application::Instance()->getPreferences(), DataStorageMode::ForceOutOfCore, opts.oocThreshold); Result r = PrepareBenchmarkInput(dsOoc, spec, geom, cellAmName, inArrayName, opts.targetVoxels, opts.minZSlices, opts.force2D, DataStorageMode::ForceOutOfCore); if(r.invalid()) { @@ -450,7 +420,7 @@ BenchmarkRow RunFilterBenchmark(const BenchmarkFilterSpec& spec, const SweepOpti if(built) { const std::unordered_set baseOoc = SnapshotIds(dsOoc); - const BenchPreferencesSentinel sentinel(DataStorageMode::ForceOutOfCore, opts.oocThreshold); + const ip_bench::BenchmarkPreferenceSnapshot sentinel(*Application::Instance()->getPreferences(), DataStorageMode::ForceOutOfCore, opts.oocThreshold); std::string note; row.newOocMs = RunConfig(filterList, spec.newFilterUuid, dsOoc, baseOoc, spec, geom, oocInput, outName, opts.warmup, opts.repeats, opts.projectionAxis, note); if(!note.empty()) diff --git a/src/Plugins/ImageProcessing/benchmark/WriteImageBenchmark.cpp b/src/Plugins/ImageProcessing/benchmark/WriteImageBenchmark.cpp index 7e971c3062..d98cff495e 100644 --- a/src/Plugins/ImageProcessing/benchmark/WriteImageBenchmark.cpp +++ b/src/Plugins/ImageProcessing/benchmark/WriteImageBenchmark.cpp @@ -1,3 +1,4 @@ +#include "BenchmarkPreferenceSnapshot.hpp" #include "ImageProcessing/Filters/WriteImageFilter.hpp" #include "simplnx/Common/Result.hpp" @@ -29,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -92,35 +94,6 @@ struct TimingRow OocCounters counterDelta; }; -class PreferencesSentinel -{ -public: - explicit PreferencesSentinel(DataStorageMode storageMode) - : m_Preferences(Application::Instance()->getPreferences()) - , m_OriginalMode(m_Preferences->dataStorageMode()) - , m_OriginalLargeDataSize(m_Preferences->valueAs(Preferences::k_LargeDataSize_Key)) - { - m_Preferences->setDataStorageMode(storageMode); - m_Preferences->setValue(Preferences::k_LargeDataSize_Key, int64{0}); - } - - ~PreferencesSentinel() - { - m_Preferences->setDataStorageMode(m_OriginalMode); - m_Preferences->setValue(Preferences::k_LargeDataSize_Key, m_OriginalLargeDataSize); - } - - PreferencesSentinel(const PreferencesSentinel&) = delete; - PreferencesSentinel(PreferencesSentinel&&) = delete; - PreferencesSentinel& operator=(const PreferencesSentinel&) = delete; - PreferencesSentinel& operator=(PreferencesSentinel&&) = delete; - -private: - Preferences* m_Preferences = nullptr; - DataStorageMode m_OriginalMode = DataStorageMode::Adaptive; - int64 m_OriginalLargeDataSize = 0; -}; - std::string FirstError(const Result<>& result) { return result.errors().empty() ? "operation failed without an error message" : result.errors().front().message; @@ -275,7 +248,7 @@ Result<> BuildInput(DataStructure& dataStructure, DataStorageMode storageMode) std::shared_ptr> store; { - PreferencesSentinel preferences(storageMode); + ip_bench::BenchmarkPreferenceSnapshot preferences(*Application::Instance()->getPreferences(), storageMode); store = DataStoreUtilities::CreateDataStore(dataStructure, k_InputArrayPath, {k_DimZ, k_DimY, k_DimX}, {1}); } if(store == nullptr) diff --git a/src/Plugins/ImageProcessing/test/CMakeLists.txt b/src/Plugins/ImageProcessing/test/CMakeLists.txt index 2a84e563b5..20cd3b77da 100644 --- a/src/Plugins/ImageProcessing/test/CMakeLists.txt +++ b/src/Plugins/ImageProcessing/test/CMakeLists.txt @@ -156,3 +156,17 @@ foreach(example_name IN LISTS ${PLUGIN_NAME}_EXAMPLE_PIPELINE_NAMES) endforeach() create_pipeline_tests(PLUGIN_NAME ${PLUGIN_NAME} PIPELINE_LIST ${PREBUILT_PIPELINE_NAMES}) + +# VerifyExamplePipelineCitations.py resolves every citation over the network (doi.org and publisher sites), +# so it depends on external services and on each agent's certificate store. It is not registered with CTest; +# run it manually when pipeline citations change. + +target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${simplnx_SOURCE_DIR}/test/ProfileIsolationContract.cpp") + +add_test(NAME "${PLUGIN_NAME}_test_profile_isolation" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -DPROFILE_RUNNER_ONLY=TRUE + -P "${simplnx_SOURCE_DIR}/test/VerifyTestProfileLifecycle.cmake") +set_tests_properties("${PLUGIN_NAME}_test_profile_isolation" PROPERTIES RUN_SERIAL TRUE TIMEOUT 60) diff --git a/src/Plugins/ImageProcessing/test/ImageProcessing_test_main.cpp b/src/Plugins/ImageProcessing/test/ImageProcessing_test_main.cpp index 48425c5326..6b322c3f17 100644 --- a/src/Plugins/ImageProcessing/test/ImageProcessing_test_main.cpp +++ b/src/Plugins/ImageProcessing/test/ImageProcessing_test_main.cpp @@ -1,4 +1,11 @@ // Catch2 recommends placing these lines by themselves in a translation unit // which will help reduce unnecessary recompilations of the expensive Catch main -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_RUNNER #include + +#include "simplnx/UnitTest/OwnedCatchRunner.hpp" + +int main(int argc, char* argv[]) +{ + return nx::core::UnitTest::RunOwnedCatchSession(argc, argv); +} diff --git a/src/Plugins/ImageProcessing/test/MorphologicalWatershedFromMarkersImageFilterTest.cpp b/src/Plugins/ImageProcessing/test/MorphologicalWatershedFromMarkersImageFilterTest.cpp index fce51f6a93..82757a907f 100644 --- a/src/Plugins/ImageProcessing/test/MorphologicalWatershedFromMarkersImageFilterTest.cpp +++ b/src/Plugins/ImageProcessing/test/MorphologicalWatershedFromMarkersImageFilterTest.cpp @@ -1,11 +1,13 @@ #include "ItkGoldenTestUtils.hpp" #include "ReconstructionFilterTestUtils.hpp" +#include "WatershedPreflightTestUtils.hpp" #include "ImageProcessing/Filters/MorphologicalWatershedFromMarkersImageFilter.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/EmptyDataStore.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/DataStructure/IDataStore.hpp" @@ -382,3 +384,128 @@ TEST_CASE("ImageProcessing::MorphologicalWatershedFromMarkersImageFilter: ITK re UnitTest::CheckArraysInheritTupleDims(ds); } + +TEST_CASE("ImageProcessing::MorphologicalWatershedFromMarkersImageFilter: preflight rejects missing temporary provider", "[ImageProcessing][MorphologicalWatershedFromMarkersImageFilter]") +{ + UnitTest::LoadPlugins(); + const auto application = Application::GetOrCreateInstance(); + auto& collection = application->getIOCollection(); + const DataIOCollection::map_type originalManagers(collection.begin(), collection.end()); + const bool originalTemporaryCapability = collection.hasTemporaryRecordStoreCapability(); + const bool originalSortCapability = collection.hasExternalSortCapability(); + const bool originalImportCapability = collection.anyManagerFinalizesImport(); + const auto originalNumericFormats = collection.getFormatNames(); + const auto originalDisplayNames = collection.getFormatDisplayNames(); + const std::string plannedFormat = "HDF5"; + REQUIRE(collection.getManager(plannedFormat) != nullptr); + const auto originalInMemoryManager = collection.getManager(Preferences::k_InMemoryFormat.str()); + REQUIRE(originalInMemoryManager != nullptr); + + SECTION("planned grayscale reaches the missing-provider guard") + { + watershed_preflight_test::ScopedMissingTemporaryProvider guard(collection); + REQUIRE_FALSE(collection.hasTemporaryRecordStoreCapability()); + REQUIRE(collection.getManager(Preferences::k_InMemoryFormat.str()) == originalInMemoryManager); + REQUIRE(collection.hasDataStoreCreationFunction(plannedFormat)); + + DataStructure dataStructure; + const DataPath geomPath({"Planned Image"}); + const auto cellPath = geomPath.createChildPath("CellData"); + const auto inputPath = cellPath.createChildPath("Grayscale"); + const auto markerPath = cellPath.createChildPath("Markers"); + const auto outputPath = cellPath.createChildPath("Output"); + const ShapeType tupleShape{1, 2, 3}; + const ShapeType componentShape{1}; + auto* imageGeom = ImageGeom::Create(dataStructure, geomPath.getTargetName()); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({3, 2, 1}); + imageGeom->setSpacing({1.0F, 1.0F, 1.0F}); + imageGeom->setOrigin({0.0F, 0.0F, 0.0F}); + auto* cellData = AttributeMatrix::Create(dataStructure, cellPath.getTargetName(), tupleShape, imageGeom->getId()); + REQUIRE(cellData != nullptr); + imageGeom->setCellData(*cellData); + + // HDF5 identifies a registered metadata-only format here. Neither input contains HDF5 values. + auto inputStoreResult = EmptyDataStore::Create(tupleShape, componentShape, plannedFormat); + SIMPLNX_RESULT_REQUIRE_VALID(inputStoreResult); + const auto* inputStore = inputStoreResult.value().get(); + auto* inputArray = Int16Array::Create(dataStructure, inputPath.getTargetName(), std::move(inputStoreResult.value()), cellData->getId()); + REQUIRE(inputArray != nullptr); + auto markerStoreResult = EmptyDataStore::Create(tupleShape, componentShape, ""); + SIMPLNX_RESULT_REQUIRE_VALID(markerStoreResult); + const auto* markerStore = markerStoreResult.value().get(); + auto* markerArray = UInt32Array::Create(dataStructure, markerPath.getTargetName(), std::move(markerStoreResult.value()), cellData->getId()); + REQUIRE(markerArray != nullptr); + + const auto checkInputs = [&] { + REQUIRE(dataStructure.getDataAs(inputPath) == inputArray); + REQUIRE(inputArray->getIDataStore() == inputStore); + REQUIRE(inputArray->getDataType() == DataType::int16); + REQUIRE(inputArray->getTupleShape() == tupleShape); + REQUIRE(inputArray->getComponentShape() == componentShape); + REQUIRE(inputArray->getStoreType() == IDataStore::StoreType::Empty); + REQUIRE(inputStore->getPlannedStoreType() == IDataStore::StoreType::OutOfCore); + REQUIRE(inputArray->getDataFormat() == plannedFormat); + REQUIRE(dataStructure.getDataAs(markerPath) == markerArray); + REQUIRE(markerArray->getIDataStore() == markerStore); + REQUIRE(markerArray->getDataType() == DataType::uint32); + REQUIRE(markerArray->getTupleShape() == tupleShape); + REQUIRE(markerArray->getComponentShape() == componentShape); + REQUIRE(markerArray->getStoreType() == IDataStore::StoreType::Empty); + REQUIRE(markerStore->getPlannedStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(markerArray->getDataFormat().empty()); + }; + checkInputs(); + REQUIRE(dataStructure.getData(outputPath) == nullptr); + const auto originalIds = dataStructure.getAllDataObjectIds(); + + WSFilter filter; + Arguments args; + args.insertOrAssign(WSFilter::k_InputImageGeomPath_Key, std::make_any(geomPath)); + args.insertOrAssign(WSFilter::k_InputImageDataPath_Key, std::make_any(inputPath)); + args.insertOrAssign(WSFilter::k_MarkerImageDataPath_Key, std::make_any(markerPath)); + args.insertOrAssign(WSFilter::k_MarkWatershedLine_Key, std::make_any(true)); + args.insertOrAssign(WSFilter::k_FullyConnected_Key, std::make_any(false)); + args.insertOrAssign(WSFilter::k_OutputImageArrayName_Key, std::make_any(outputPath.getTargetName())); + const auto result = filter.preflight(dataStructure, args); + REQUIRE(result.outputActions.invalid()); + REQUIRE(result.outputActions.errors().size() == 1); + REQUIRE(result.outputActions.errors().front().code == -79047); + REQUIRE(result.outputActions.errors().front().message.find(inputPath.toString()) != std::string::npos); + REQUIRE(result.outputActions.warnings().empty()); + REQUIRE(dataStructure.getData(outputPath) == nullptr); + REQUIRE(dataStructure.getAllDataObjectIds() == originalIds); + checkInputs(); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + SECTION("manager slots restore after a controlled throw") + { + struct ControlledUnwind + { + }; + bool reachedGuardedBody = false; + bool caughtControlledThrow = false; + try + { + watershed_preflight_test::ScopedMissingTemporaryProvider guard(collection); + REQUIRE_FALSE(collection.hasTemporaryRecordStoreCapability()); + REQUIRE(collection.getManager(Preferences::k_InMemoryFormat.str()) == originalInMemoryManager); + reachedGuardedBody = true; + throw ControlledUnwind{}; + } catch(const ControlledUnwind&) + { + caughtControlledThrow = true; + } + REQUIRE(reachedGuardedBody); + REQUIRE(caughtControlledThrow); + } + + const DataIOCollection::map_type restoredManagers(collection.begin(), collection.end()); + REQUIRE(restoredManagers == originalManagers); + REQUIRE(collection.hasTemporaryRecordStoreCapability() == originalTemporaryCapability); + REQUIRE(collection.hasExternalSortCapability() == originalSortCapability); + REQUIRE(collection.anyManagerFinalizesImport() == originalImportCapability); + REQUIRE(collection.getFormatNames() == originalNumericFormats); + REQUIRE(collection.getFormatDisplayNames() == originalDisplayNames); +} diff --git a/src/Plugins/ImageProcessing/test/MorphologicalWatershedImageFilterTest.cpp b/src/Plugins/ImageProcessing/test/MorphologicalWatershedImageFilterTest.cpp index 582263e8f2..38ad27ccab 100644 --- a/src/Plugins/ImageProcessing/test/MorphologicalWatershedImageFilterTest.cpp +++ b/src/Plugins/ImageProcessing/test/MorphologicalWatershedImageFilterTest.cpp @@ -1,11 +1,15 @@ #include "ItkGoldenTestUtils.hpp" #include "ReconstructionFilterTestUtils.hpp" +#include "WatershedPreflightTestUtils.hpp" #include "ImageProcessing/Filters/MorphologicalWatershedImageFilter.hpp" +#include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/AbstractDataStore.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/EmptyDataStore.hpp" +#include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/Filter/IFilter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" @@ -19,6 +23,7 @@ #include #include +#include #include using namespace nx::core; @@ -161,3 +166,91 @@ TEST_CASE("ImageProcessing::MorphologicalWatershedImageFilter: ITK real-image go args.insertOrAssign(MorphologicalWatershedImageFilter::k_MarkWatershedLine_Key, std::make_any(false)); }); } + +TEST_CASE("ImageProcessing::MorphologicalWatershedImageFilter: preflight rejects missing temporary provider", "[ImageProcessing][MorphologicalWatershedImageFilter]") +{ + UnitTest::LoadPlugins(); + const auto application = Application::GetOrCreateInstance(); + auto& collection = application->getIOCollection(); + const DataIOCollection::map_type originalManagers(collection.begin(), collection.end()); + const bool originalTemporaryCapability = collection.hasTemporaryRecordStoreCapability(); + const bool originalSortCapability = collection.hasExternalSortCapability(); + const bool originalImportCapability = collection.anyManagerFinalizesImport(); + const auto originalNumericFormats = collection.getFormatNames(); + const auto originalDisplayNames = collection.getFormatDisplayNames(); + const std::string plannedFormat = "HDF5"; + REQUIRE(collection.getManager(plannedFormat) != nullptr); + const auto originalInMemoryManager = collection.getManager(Preferences::k_InMemoryFormat.str()); + REQUIRE(originalInMemoryManager != nullptr); + + { + watershed_preflight_test::ScopedMissingTemporaryProvider guard(collection); + REQUIRE_FALSE(collection.hasTemporaryRecordStoreCapability()); + REQUIRE(collection.getManager(Preferences::k_InMemoryFormat.str()) == originalInMemoryManager); + REQUIRE(collection.hasDataStoreCreationFunction(plannedFormat)); + + DataStructure dataStructure; + const DataPath geomPath({"Planned Image"}); + const auto cellPath = geomPath.createChildPath("CellData"); + const auto inputPath = cellPath.createChildPath("Grayscale"); + const auto outputPath = cellPath.createChildPath("Output"); + const ShapeType tupleShape{1, 2, 3}; + const ShapeType componentShape{1}; + auto* imageGeom = ImageGeom::Create(dataStructure, geomPath.getTargetName()); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({3, 2, 1}); + imageGeom->setSpacing({1.0F, 1.0F, 1.0F}); + imageGeom->setOrigin({0.0F, 0.0F, 0.0F}); + auto* cellData = AttributeMatrix::Create(dataStructure, cellPath.getTargetName(), tupleShape, imageGeom->getId()); + REQUIRE(cellData != nullptr); + imageGeom->setCellData(*cellData); + + // HDF5 identifies a registered metadata-only format here. The input contains no HDF5 values. + auto inputStoreResult = EmptyDataStore::Create(tupleShape, componentShape, plannedFormat); + SIMPLNX_RESULT_REQUIRE_VALID(inputStoreResult); + const auto* inputStore = inputStoreResult.value().get(); + auto* inputArray = Int16Array::Create(dataStructure, inputPath.getTargetName(), std::move(inputStoreResult.value()), cellData->getId()); + REQUIRE(inputArray != nullptr); + + const auto checkInput = [&] { + REQUIRE(dataStructure.getDataAs(inputPath) == inputArray); + REQUIRE(inputArray->getIDataStore() == inputStore); + REQUIRE(inputArray->getDataType() == DataType::int16); + REQUIRE(inputArray->getTupleShape() == tupleShape); + REQUIRE(inputArray->getComponentShape() == componentShape); + REQUIRE(inputArray->getStoreType() == IDataStore::StoreType::Empty); + REQUIRE(inputStore->getPlannedStoreType() == IDataStore::StoreType::OutOfCore); + REQUIRE(inputArray->getDataFormat() == plannedFormat); + }; + checkInput(); + REQUIRE(dataStructure.getData(outputPath) == nullptr); + const auto originalIds = dataStructure.getAllDataObjectIds(); + + WSFilter filter; + Arguments args; + args.insertOrAssign(WSFilter::k_InputImageGeomPath_Key, std::make_any(geomPath)); + args.insertOrAssign(WSFilter::k_InputImageDataPath_Key, std::make_any(inputPath)); + args.insertOrAssign(WSFilter::k_OutputImageArrayName_Key, std::make_any(outputPath.getTargetName())); + args.insertOrAssign(WSFilter::k_Level_Key, std::make_any(0.0)); + args.insertOrAssign(WSFilter::k_MarkWatershedLine_Key, std::make_any(true)); + args.insertOrAssign(WSFilter::k_FullyConnected_Key, std::make_any(false)); + const auto result = filter.preflight(dataStructure, args); + REQUIRE(result.outputActions.invalid()); + REQUIRE(result.outputActions.errors().size() == 1); + REQUIRE(result.outputActions.errors().front().code == -79048); + REQUIRE(result.outputActions.errors().front().message.find(inputPath.toString()) != std::string::npos); + REQUIRE(result.outputActions.warnings().empty()); + REQUIRE(dataStructure.getData(outputPath) == nullptr); + REQUIRE(dataStructure.getAllDataObjectIds() == originalIds); + checkInput(); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + const DataIOCollection::map_type restoredManagers(collection.begin(), collection.end()); + REQUIRE(restoredManagers == originalManagers); + REQUIRE(collection.hasTemporaryRecordStoreCapability() == originalTemporaryCapability); + REQUIRE(collection.hasExternalSortCapability() == originalSortCapability); + REQUIRE(collection.anyManagerFinalizesImport() == originalImportCapability); + REQUIRE(collection.getFormatNames() == originalNumericFormats); + REQUIRE(collection.getFormatDisplayNames() == originalDisplayNames); +} diff --git a/src/Plugins/ImageProcessing/test/WatershedPreflightTestUtils.hpp b/src/Plugins/ImageProcessing/test/WatershedPreflightTestUtils.hpp new file mode 100644 index 0000000000..065b7498f3 --- /dev/null +++ b/src/Plugins/ImageProcessing/test/WatershedPreflightTestUtils.hpp @@ -0,0 +1,114 @@ +#pragma once + +#include "simplnx/Core/Preferences.hpp" +#include "simplnx/DataStructure/IO/Generic/DataIOCollection.hpp" +#include "simplnx/DataStructure/IO/Generic/IDataIOManager.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @namespace watershed_preflight_test + * @brief Contains metadata-only fixtures for watershed preflight checks. + */ +namespace watershed_preflight_test +{ +/** + * @class ScopedMissingTemporaryProvider + * @brief Replaces optional managers with metadata-only managers until scope exit. + * + * The fixture keeps each map key and the built-in in-memory manager unchanged. It allocates all replacements before the first swap. + * @note The collection must outlive this guard. No other caller can access or modify the collection while this guard exists. + */ +class ScopedMissingTemporaryProvider +{ +public: + /** + * @brief Removes temporary-record capability without creating or deleting registry entries. + * @param collection Registry to isolate for one serial preflight check. + * @throws std::bad_alloc If replacement allocation fails before any registry slot changes. + */ + explicit ScopedMissingTemporaryProvider(nx::core::DataIOCollection& collection) + { + m_Replacements.reserve(static_cast(std::distance(collection.begin(), collection.end()))); + for(auto& [name, manager] : collection) + { + if(name != nx::core::Preferences::k_InMemoryFormat) + { + m_Replacements.push_back({&manager, std::make_shared(name)}); + } + } + + // All remaining operations are no-throw swaps, so construction cannot leave a partially changed registry. + for(auto& replacement : m_Replacements) + { + replacement.slot->swap(replacement.manager); + } + } + + /** + * @brief Restores the original manager objects without allocation or changes to map keys. + */ + ~ScopedMissingTemporaryProvider() noexcept + { + for(auto& replacement : m_Replacements) + { + replacement.slot->swap(replacement.manager); + } + } + + ScopedMissingTemporaryProvider(const ScopedMissingTemporaryProvider&) = delete; + ScopedMissingTemporaryProvider(ScopedMissingTemporaryProvider&&) = delete; + ScopedMissingTemporaryProvider& operator=(const ScopedMissingTemporaryProvider&) = delete; + ScopedMissingTemporaryProvider& operator=(ScopedMissingTemporaryProvider&&) = delete; + +private: + /** + * @class MetadataOnlyManager + * @brief Registers a numeric format that rejects all value-store creation. + */ + class MetadataOnlyManager : public nx::core::IDataIOManager + { + public: + /** + * @brief Makes the existing format available for metadata validation only. + * @param format Existing registry key to preserve. + */ + explicit MetadataOnlyManager(std::string format) + : m_Format(std::move(format)) + { + addDataStoreCreationFnc(m_Format, + [](nx::core::DataType, const nx::core::ShapeType&, const nx::core::ShapeType&, const std::optional&, nx::core::DataStoreInitializationMode) + -> std::unique_ptr { throw std::runtime_error("Watershed preflight must not create a value store from its metadata-only format"); }); + } + + ~MetadataOnlyManager() override = default; + MetadataOnlyManager(const MetadataOnlyManager&) = default; + MetadataOnlyManager(MetadataOnlyManager&&) = default; + MetadataOnlyManager& operator=(const MetadataOnlyManager&) = default; + MetadataOnlyManager& operator=(MetadataOnlyManager&&) = default; + + std::string formatName() const override + { + return m_Format; + } + + private: + std::string m_Format; + }; + + struct Replacement + { + std::shared_ptr* slot = nullptr; + std::shared_ptr manager; + }; + + std::vector m_Replacements; +}; +} // namespace watershed_preflight_test diff --git a/src/Plugins/ImageProcessing/test/ZeroCrossingEngineTest.cpp b/src/Plugins/ImageProcessing/test/ZeroCrossingEngineTest.cpp index 3a5352db73..65056593f4 100644 --- a/src/Plugins/ImageProcessing/test/ZeroCrossingEngineTest.cpp +++ b/src/Plugins/ImageProcessing/test/ZeroCrossingEngineTest.cpp @@ -1,10 +1,12 @@ #include "simplnx/Utilities/ImageProcessing/ZeroCrossingEngine.hpp" #include "simplnx/Common/Array.hpp" +#include "simplnx/Common/ScopeGuard.hpp" #include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/Filter/IFilter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" #include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/ImageProcessing/WorkingMemory.hpp" @@ -706,3 +708,64 @@ TEST_CASE("ImageProcessing::ZeroCrossingEngine: bounded 2D blocks and tiles pres } } } + +TEST_CASE("ImageProcessing::ZeroCrossingEngine: resident flag precedence has an execution witness", "[ImageProcessing][ZeroCrossingEngine][ZeroCrossingSelection]") +{ + UnitTest::LoadPlugins(); + // Low-level selector tests use raw flags to cover the otherwise conflicting force controls. + struct FlagCase + { + bool ForceInCore; + bool ForceOoc; + bool ExpectDirect; + }; + constexpr std::array cases{{ + {.ForceInCore = false, .ForceOoc = false, .ExpectDirect = true}, + {.ForceInCore = false, .ForceOoc = true, .ExpectDirect = false}, + {.ForceInCore = true, .ForceOoc = false, .ExpectDirect = true}, + {.ForceInCore = true, .ForceOoc = true, .ExpectDirect = true}, + }}; + // Equal magnitudes assign each crossing to the lower X endpoint. There are no Y or Z crossings. + constexpr std::array input{-1.0F, 1.0F, -1.0F, 1.0F}; + constexpr std::array expected{7, 3, 7, 3}; + for(const auto& testCase : cases) + { + CAPTURE(testCase.ForceInCore, testCase.ForceOoc); + const bool originalInCore = ForceInCoreAlgorithm(); + const bool originalOoc = ForceOocAlgorithm(); + const auto originalCounts = GetAlgorithmPathExecutionCounts(); + const auto restoreState = MakeScopeGuard([=]() noexcept { + ForceInCoreAlgorithm() = originalInCore; + ForceOocAlgorithm() = originalOoc; + SetAlgorithmPathExecutionCounts(originalCounts); + }); + const ScopedBudget budget(8192); + ForceInCoreAlgorithm() = testCase.ForceInCore; + ForceOocAlgorithm() = testCase.ForceOoc; + ResetAlgorithmPathExecutionCounts(); + TransferCountingDataStore inputStore(ShapeType{2, 1, 2}, ShapeType{1}, 0.0F); + TransferCountingDataStore outputStore(ShapeType{2, 1, 2}, ShapeType{1}, uint8{99}); + for(usize index = 0; index < input.size(); ++index) + { + inputStore.setValue(index, input[index]); + } + const std::atomic_bool shouldCancel{false}; + const IFilter::MessageHandler messages{}; + const auto result = ApplyZeroCrossing(inputStore, outputStore, SizeVec3{2, 1, 2}, uint8{7}, uint8{3}, shouldCancel, messages); + SIMPLNX_RESULT_REQUIRE_VALID(result); + const auto counts = GetAlgorithmPathExecutionCounts(); + CHECK(counts.InCore == (testCase.ExpectDirect ? 1 : 0)); + CHECK(counts.OutOfCore == (testCase.ExpectDirect ? 0 : 1)); + CHECK(counts.InCoreOnInMemoryStore == counts.InCore); + CHECK(counts.OutOfCoreOnInMemoryStore == counts.OutOfCore); + CHECK(counts.InCoreOnOutOfCoreStore == 0); + CHECK(counts.OutOfCoreOnOutOfCoreStore == 0); + CHECK((inputStore.readCount() == 0) == testCase.ExpectDirect); + CHECK((outputStore.writeCount() == 0) == testCase.ExpectDirect); + for(usize index = 0; index < expected.size(); ++index) + { + CAPTURE(index); + CHECK(outputStore.getValue(index) == expected[index]); + } + } +} diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFaceIPFColoring.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFaceIPFColoring.cpp index 19f1bf3444..5480a9d72d 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFaceIPFColoring.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFaceIPFColoring.cpp @@ -1,8 +1,12 @@ #include "ComputeFaceIPFColoring.hpp" +#include "OrientationTupleLookup.hpp" +#include "simplnx/Common/Result.hpp" #include "simplnx/Common/RgbColor.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" #include @@ -18,10 +22,14 @@ #include #include #include +#include #include #include +#include +#include +#include #include using namespace nx::core; @@ -157,20 +165,20 @@ class CalculateFaceIPFColorsImpl dEuler[1] = m_FeatureEulerAngles[3 * firstFeatureIdx + 1]; dEuler[2] = m_FeatureEulerAngles[3 * firstFeatureIdx + 2]; refDir[0] = m_FaceNormals[3 * faceIdx]; - refDir[1] = m_FaceNormals[3 * faceIdx + 1]; - refDir[2] = m_FaceNormals[3 * faceIdx + 2]; + refDir[1] = m_FaceNormals[(3 * faceIdx) + 1]; + refDir[2] = m_FaceNormals[(3 * faceIdx) + 2]; argb = orientationOps[currentLaueIndex]->generateIPFColor(dEuler, refDir, false, m_ColorKey); m_FirstColors[3 * faceIdx] = RgbColor::dRed(argb); - m_FirstColors[3 * faceIdx + 1] = RgbColor::dGreen(argb); - m_FirstColors[3 * faceIdx + 2] = RgbColor::dBlue(argb); + m_FirstColors[(3 * faceIdx) + 1] = RgbColor::dGreen(argb); + m_FirstColors[(3 * faceIdx) + 2] = RgbColor::dBlue(argb); } } else // A face side without a positive Phase receives black. { m_FirstColors[3 * faceIdx] = 0; - m_FirstColors[3 * faceIdx + 1] = 0; - m_FirstColors[3 * faceIdx + 2] = 0; + m_FirstColors[(3 * faceIdx) + 1] = 0; + m_FirstColors[(3 * faceIdx) + 2] = 0; } // The second face side uses the opposite normal direction. @@ -183,20 +191,20 @@ class CalculateFaceIPFColorsImpl dEuler[1] = m_FeatureEulerAngles[3 * secondFeatureIdx + 1]; dEuler[2] = m_FeatureEulerAngles[3 * secondFeatureIdx + 2]; refDir[0] = -m_FaceNormals[3 * faceIdx]; - refDir[1] = -m_FaceNormals[3 * faceIdx + 1]; - refDir[2] = -m_FaceNormals[3 * faceIdx + 2]; + refDir[1] = -m_FaceNormals[(3 * faceIdx) + 1]; + refDir[2] = -m_FaceNormals[(3 * faceIdx) + 2]; argb = orientationOps[currentLaueIndex]->generateIPFColor(dEuler, refDir, false, m_ColorKey); m_SecondColors[3 * faceIdx] = RgbColor::dRed(argb); - m_SecondColors[3 * faceIdx + 1] = RgbColor::dGreen(argb); - m_SecondColors[3 * faceIdx + 2] = RgbColor::dBlue(argb); + m_SecondColors[(3 * faceIdx) + 1] = RgbColor::dGreen(argb); + m_SecondColors[(3 * faceIdx) + 2] = RgbColor::dBlue(argb); } } else { m_SecondColors[3 * faceIdx] = 0; - m_SecondColors[3 * faceIdx + 1] = 0; - m_SecondColors[3 * faceIdx + 2] = 0; + m_SecondColors[(3 * faceIdx) + 1] = 0; + m_SecondColors[(3 * faceIdx) + 2] = 0; } } } @@ -211,6 +219,213 @@ class CalculateFaceIPFColorsImpl } }; +namespace +{ +Result<> CalculateFaceIPFColorsInBlocks(const Int32Array& faceLabels, const Int32Array& phases, const Float64Array& faceNormals, const Float32Array& eulerAngles, const UInt32Array& structures, + UInt8Array& firstColors, UInt8Array& secondColors, const ComputeFaceIPFColoringInputValues& inputValues, const std::atomic_bool& shouldCancel) +{ + const usize numTriangles = faceLabels.getNumberOfTuples(); + if(shouldCancel || numTriangles == 0) + { + return {}; + } + using namespace orientation::detail; + constexpr usize faceBytes = (2 * sizeof(int32)) + (3 * sizeof(float64)) + (6 * sizeof(uint8)); + auto planResult = + PlanLookups({.FaceCount = numTriangles, .FeatureCount = phases.getNumberOfTuples(), .EnsembleCount = structures.getNumberOfTuples(), .OrientationComponents = 3, .FaceBytes = faceBytes}, -24359); + if(planResult.invalid()) + { + return ConvertResult(std::move(planResult)); + } + // This owner precedes every buffer, so the reservation outlives their payloads. + auto plan = std::move(planResult.value()); + const usize bufferTriangles = plan.FaceCapacity; + const bool wholeTables = plan.CacheWholeTables; + Result<> result; + std::vector featurePhases(wholeTables ? phases.getSize() : bufferTriangles * 2); + std::vector featureEulerAngles(wholeTables ? eulerAngles.getSize() : bufferTriangles * 6); + std::vector crystalStructures(wholeTables ? structures.getSize() : bufferTriangles * 2); + std::vector lookupOrder(wholeTables ? 0 : bufferTriangles * 2); + std::vector phasePage(plan.PageCapacity); + std::vector eulerPage(plan.PageCapacity * 3); + std::vector structurePage(plan.PageCapacity); + if(wholeTables) + { + result = MergeResults(std::move(result), ReadLookupTable(phases.getDataStoreRef(), featurePhases, -24350, inputValues.FeaturePhasesArrayPath)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), ReadLookupTable(eulerAngles.getDataStoreRef(), featureEulerAngles, -24351, inputValues.FeatureEulerAnglesArrayPath)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), ReadLookupTable(structures.getDataStoreRef(), crystalStructures, -24352, inputValues.CrystalStructuresArrayPath)); + if(result.invalid()) + { + return result; + } + } + + std::vector faceLabelsBuffer(bufferTriangles * 2); + std::vector faceNormalsBuffer(bufferTriangles * 3); + std::vector firstColorsBuffer(bufferTriangles * 3); + std::vector secondColorsBuffer(bufferTriangles * 3); + const std::vector orientationOps = ebsdlib::LaueOps::GetAllOrientationOps(); + for(usize start = 0; start < numTriangles;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(bufferTriangles, numTriangles - start); + result = MergeResults(std::move(result), faceLabels.getDataStoreRef().copyIntoBuffer(start * 2, nonstd::span(faceLabelsBuffer.data(), count * 2))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-24353, fmt::format("Error reading face labels for face IPF coloring at triangle {} ({} triangles).", start, count))); + } + result = MergeResults(std::move(result), faceNormals.getDataStoreRef().copyIntoBuffer(start * 3, nonstd::span(faceNormalsBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-24354, fmt::format("Error reading face normals for face IPF coloring at triangle {} ({} triangles).", start, count))); + } + // Unsupported Laue indices leave the existing output unchanged in the in-core path. + result = MergeResults(std::move(result), firstColors.getDataStoreRef().copyIntoBuffer(start * 3, nonstd::span(firstColorsBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-24355, fmt::format("Error reading first face IPF colors at triangle {} ({} triangles).", start, count))); + } + result = MergeResults(std::move(result), secondColors.getDataStoreRef().copyIntoBuffer(start * 3, nonstd::span(secondColorsBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-24356, fmt::format("Error reading second face IPF colors at triangle {} ({} triangles).", start, count))); + } + + if(!wholeTables) + { + const nonstd::span labels(faceLabelsBuffer.data(), count * 2); + result = MergeResults(std::move(result), + GatherLookupTuples(phases.getDataStoreRef(), labels, lookupOrder, featurePhases, phasePage, shouldCancel, -24350, inputValues.FeaturePhasesArrayPath)); + if(result.invalid() || shouldCancel) + { + return result; + } + result = MergeResults(std::move(result), GatherLookupTuples(eulerAngles.getDataStoreRef(), labels, lookupOrder, featureEulerAngles, eulerPage, shouldCancel, -24351, + inputValues.FeatureEulerAnglesArrayPath, true)); + if(result.invalid() || shouldCancel) + { + return result; + } + result = MergeResults(std::move(result), GatherLookupTuples(structures.getDataStoreRef(), {featurePhases.data(), count * 2}, lookupOrder, crystalStructures, structurePage, + shouldCancel, -24352, inputValues.CrystalStructuresArrayPath)); + if(result.invalid() || shouldCancel) + { + return result; + } + } + + // Preserve the in-core color calculations, including the opposite second-side normal. + + std::array refDir{}; + std::array dEuler{}; + Rgba argb = 0x00000000; + + int32 firstFeatureIdx = 0; + int32 secondFeatureIdx = 0; + int32 firstFeaturePhaseIdx = 0; + int32 secondFeaturePhaseIdx = 0; + for(usize faceIdx = 0; faceIdx < count; faceIdx++) + { + firstFeatureIdx = faceLabelsBuffer[2 * faceIdx]; + secondFeatureIdx = faceLabelsBuffer[(2 * faceIdx) + 1]; + if(firstFeatureIdx > 0) + { + firstFeaturePhaseIdx = featurePhases[wholeTables ? static_cast(firstFeatureIdx) : 2 * faceIdx]; + } + else + { + firstFeaturePhaseIdx = 0; + } + + if(secondFeatureIdx > 0) + { + secondFeaturePhaseIdx = featurePhases[wholeTables ? static_cast(secondFeatureIdx) : (2 * faceIdx) + 1]; + } + else + { + secondFeaturePhaseIdx = 0; + } + + if(firstFeaturePhaseIdx > 0) + { + const uint32 currentLaueIndex = crystalStructures[wholeTables ? static_cast(firstFeaturePhaseIdx) : 2 * faceIdx]; + if(currentLaueIndex < orientationOps.size()) + { + const usize firstEuler = 3 * (wholeTables ? static_cast(firstFeatureIdx) : 2 * faceIdx); + dEuler[0] = featureEulerAngles[firstEuler]; + dEuler[1] = featureEulerAngles[firstEuler + 1]; + dEuler[2] = featureEulerAngles[firstEuler + 2]; + refDir[0] = faceNormalsBuffer[3 * faceIdx]; + refDir[1] = faceNormalsBuffer[(3 * faceIdx) + 1]; + refDir[2] = faceNormalsBuffer[(3 * faceIdx) + 2]; + + argb = orientationOps[currentLaueIndex]->generateIPFColor(dEuler.data(), refDir.data(), false, inputValues.ColorKey); + firstColorsBuffer[3 * faceIdx] = RgbColor::dRed(argb); + firstColorsBuffer[(3 * faceIdx) + 1] = RgbColor::dGreen(argb); + firstColorsBuffer[(3 * faceIdx) + 2] = RgbColor::dBlue(argb); + } + } + else // A face side without a positive Phase receives black. + { + firstColorsBuffer[3 * faceIdx] = 0; + firstColorsBuffer[(3 * faceIdx) + 1] = 0; + firstColorsBuffer[(3 * faceIdx) + 2] = 0; + } + + // The second face side uses the opposite normal direction. + if(secondFeaturePhaseIdx > 0) + { + const uint32 currentLaueIndex = crystalStructures[wholeTables ? static_cast(secondFeaturePhaseIdx) : (2 * faceIdx) + 1]; + if(currentLaueIndex < orientationOps.size()) + { + const usize secondEuler = 3 * (wholeTables ? static_cast(secondFeatureIdx) : (2 * faceIdx) + 1); + dEuler[0] = featureEulerAngles[secondEuler]; + dEuler[1] = featureEulerAngles[secondEuler + 1]; + dEuler[2] = featureEulerAngles[secondEuler + 2]; + refDir[0] = -faceNormalsBuffer[3 * faceIdx]; + refDir[1] = -faceNormalsBuffer[(3 * faceIdx) + 1]; + refDir[2] = -faceNormalsBuffer[(3 * faceIdx) + 2]; + + argb = orientationOps[currentLaueIndex]->generateIPFColor(dEuler.data(), refDir.data(), false, inputValues.ColorKey); + secondColorsBuffer[3 * faceIdx] = RgbColor::dRed(argb); + secondColorsBuffer[(3 * faceIdx) + 1] = RgbColor::dGreen(argb); + secondColorsBuffer[(3 * faceIdx) + 2] = RgbColor::dBlue(argb); + } + } + else + { + secondColorsBuffer[3 * faceIdx] = 0; + secondColorsBuffer[(3 * faceIdx) + 1] = 0; + secondColorsBuffer[(3 * faceIdx) + 2] = 0; + } + } + result = MergeResults(std::move(result), firstColors.getDataStoreRef().copyFromBuffer(start * 3, nonstd::span(firstColorsBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-24357, fmt::format("Error writing first face IPF colors at triangle {} ({} triangles).", start, count))); + } + result = MergeResults(std::move(result), secondColors.getDataStoreRef().copyFromBuffer(start * 3, nonstd::span(secondColorsBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-24358, fmt::format("Error writing second face IPF colors at triangle {} ({} triangles).", start, count))); + } + start += count; + } + return result; +} +} // namespace + // ----------------------------------------------------------------------------- ComputeFaceIPFColoring::ComputeFaceIPFColoring(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, ComputeFaceIPFColoringInputValues* inputValues) @@ -243,6 +458,16 @@ Result<> ComputeFaceIPFColoring::operator()() return validationResult; } + const bool usesOutOfCoreStore = + AnyOutOfCore({&faceLabelsArrayRef, &faceNormalsArrayRef, &featureEulerAnglesArrayRef, &featurePhasesArrayRef, &crystalStructuresArrayRef, &firstIpfColorsArrayRef, &secondIpfColorsArrayRef}); + const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + return CalculateFaceIPFColorsInBlocks(faceLabelsArrayRef, featurePhasesArrayRef, faceNormalsArrayRef, featureEulerAnglesArrayRef, crystalStructuresArrayRef, firstIpfColorsArrayRef, + secondIpfColorsArrayRef, *m_InputValues, m_ShouldCancel); + } + typename IParallelAlgorithm::AlgorithmArrays algArrays; algArrays.push_back(&faceLabelsArrayRef); algArrays.push_back(&faceNormalsArrayRef); diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFeatureFaceMisorientation.cpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFeatureFaceMisorientation.cpp index 0c937505e5..f7febaf746 100644 --- a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFeatureFaceMisorientation.cpp +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/ComputeFeatureFaceMisorientation.cpp @@ -1,15 +1,26 @@ #include "ComputeFeatureFaceMisorientation.hpp" +#include "OrientationTupleLookup.hpp" #include "simplnx/Common/Constants.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" #include #include +#include +#include #include #include +#include +#include +#include +#include +#include using LaueOpsShPtrType = std::shared_ptr; using LaueOpsContainer = std::vector; @@ -144,12 +155,12 @@ class ComputeFeatureMisorientationPerTriangleImpl float32 quat1 = m_FeatureAvgQuats[frontFeatureIdx * 4 + 1]; float32 quat2 = m_FeatureAvgQuats[frontFeatureIdx * 4 + 2]; float32 quat3 = m_FeatureAvgQuats[frontFeatureIdx * 4 + 3]; - ebsdlib::QuatD q1(quat0, quat1, quat2, quat3); + const ebsdlib::QuatD q1(quat0, quat1, quat2, quat3); quat0 = m_FeatureAvgQuats[backFeatureIdx * 4]; quat1 = m_FeatureAvgQuats[backFeatureIdx * 4 + 1]; quat2 = m_FeatureAvgQuats[backFeatureIdx * 4 + 2]; quat3 = m_FeatureAvgQuats[backFeatureIdx * 4 + 3]; - ebsdlib::QuatD q2(quat0, quat1, quat2, quat3); + const ebsdlib::QuatD q2(quat0, quat1, quat2, quat3); ebsdlib::AxisAngleDType axisAngle = m_LaueOrientationOps[currentLaueIndex]->calculateMisorientation(q1, q2); m_Misorientations.setValue(triangleIdx, static_cast(axisAngle[3] * Constants::k_180OverPiD)); continue; @@ -169,6 +180,164 @@ class ComputeFeatureMisorientationPerTriangleImpl } }; +namespace +{ +Result<> ComputeFeatureMisorientationInBlocks(const Int32Array& faceLabels, const Int32Array& phases, const Float32Array& quats, const UInt32Array& structures, Float32Array& misorientations, + const ComputeFeatureFaceMisorientationInputValues& inputValues, const std::atomic_bool& shouldCancel) +{ + const usize numTriangles = faceLabels.getNumberOfTuples(); + if(shouldCancel || numTriangles == 0) + { + return {}; + } + using namespace orientation::detail; + constexpr usize faceBytes = (2 * sizeof(int32)) + sizeof(float32); + auto planResult = + PlanLookups({.FaceCount = numTriangles, .FeatureCount = phases.getNumberOfTuples(), .EnsembleCount = structures.getNumberOfTuples(), .OrientationComponents = 4, .FaceBytes = faceBytes}, -98425); + if(planResult.invalid()) + { + return ConvertResult(std::move(planResult)); + } + // This owner precedes every buffer, so the reservation outlives their payloads. + auto plan = std::move(planResult.value()); + const usize bufferTriangles = plan.FaceCapacity; + const bool wholeTables = plan.CacheWholeTables; + Result<> result; + std::vector featurePhases(wholeTables ? phases.getSize() : bufferTriangles * 2); + std::vector featureAvgQuats(wholeTables ? quats.getSize() : bufferTriangles * 8); + std::vector crystalStructures(wholeTables ? structures.getSize() : bufferTriangles * 2); + std::vector lookupOrder(wholeTables ? 0 : bufferTriangles * 2); + std::vector phasePage(plan.PageCapacity); + std::vector quatPage(plan.PageCapacity * 4); + std::vector structurePage(plan.PageCapacity); + if(wholeTables) + { + result = MergeResults(std::move(result), ReadLookupTable(phases.getDataStoreRef(), featurePhases, -98420, inputValues.featurePhasesArrayPath)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), ReadLookupTable(quats.getDataStoreRef(), featureAvgQuats, -98421, inputValues.avgQuatsArrayPath)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), ReadLookupTable(structures.getDataStoreRef(), crystalStructures, -98422, inputValues.crystalStructuresArrayPath)); + if(result.invalid()) + { + return result; + } + } + + const LaueOpsContainer orientationOps = ebsdlib::LaueOps::GetAllOrientationOps(); + std::vector faceLabelsBuffer(bufferTriangles * 2); + std::vector misorientationsBuffer(bufferTriangles); + for(usize start = 0; start < numTriangles;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(bufferTriangles, numTriangles - start); + result = MergeResults(std::move(result), faceLabels.getDataStoreRef().copyIntoBuffer(start * 2, nonstd::span(faceLabelsBuffer.data(), count * 2))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-98423, fmt::format("Error reading face labels for face misorientation at triangle {} ({} triangles).", start, count))); + } + + if(!wholeTables) + { + const nonstd::span labels(faceLabelsBuffer.data(), count * 2); + result = MergeResults(std::move(result), + GatherLookupTuples(phases.getDataStoreRef(), labels, lookupOrder, featurePhases, phasePage, shouldCancel, -98420, inputValues.featurePhasesArrayPath)); + if(result.invalid() || shouldCancel) + { + return result; + } + // Only equal positive phases participate. Other phase values can be outside the ensemble range. + for(usize face = 0; face < count; ++face) + { + if(featurePhases[face * 2] <= 0 || featurePhases[face * 2] != featurePhases[(face * 2) + 1]) + { + featurePhases[face * 2] = 0; + featurePhases[(face * 2) + 1] = 0; + } + } + result = MergeResults(std::move(result), + GatherLookupTuples(quats.getDataStoreRef(), labels, lookupOrder, featureAvgQuats, quatPage, shouldCancel, -98421, inputValues.avgQuatsArrayPath, true)); + if(result.invalid() || shouldCancel) + { + return result; + } + result = MergeResults(std::move(result), GatherLookupTuples(structures.getDataStoreRef(), {featurePhases.data(), count * 2}, lookupOrder, crystalStructures, structurePage, + shouldCancel, -98422, inputValues.crystalStructuresArrayPath)); + if(result.invalid() || shouldCancel) + { + return result; + } + } + + // Keep the in-core quaternion construction, Laue operation and float conversion. + int32 frontFeatureIdx = 0; + int32 backFeatureIdx = 0; + int32 frontPhaseIdx = 0; + int32 backPhaseIdx = 0; + + for(usize triangleIdx = 0; triangleIdx < count; triangleIdx++) + { + frontFeatureIdx = faceLabelsBuffer[2 * triangleIdx]; + backFeatureIdx = faceLabelsBuffer[(2 * triangleIdx) + 1]; + if(frontFeatureIdx > 0) + { + frontPhaseIdx = featurePhases[wholeTables ? static_cast(frontFeatureIdx) : 2 * triangleIdx]; + } + else + { + frontPhaseIdx = 0; + } + if(backFeatureIdx > 0) + { + backPhaseIdx = featurePhases[wholeTables ? static_cast(backFeatureIdx) : (2 * triangleIdx) + 1]; + } + else + { + backPhaseIdx = 0; + } + if(frontPhaseIdx > 0 && frontPhaseIdx == backPhaseIdx) + { + const uint32 currentLaueIndex = crystalStructures[wholeTables ? static_cast(frontPhaseIdx) : 2 * triangleIdx]; + if(currentLaueIndex < orientationOps.size()) + { + const usize frontQuat = 4 * (wholeTables ? static_cast(frontFeatureIdx) : 2 * triangleIdx); + float32 quat0 = featureAvgQuats[frontQuat]; + float32 quat1 = featureAvgQuats[frontQuat + 1]; + float32 quat2 = featureAvgQuats[frontQuat + 2]; + float32 quat3 = featureAvgQuats[frontQuat + 3]; + const ebsdlib::QuatD q1(quat0, quat1, quat2, quat3); + const usize backQuat = 4 * (wholeTables ? static_cast(backFeatureIdx) : (2 * triangleIdx) + 1); + quat0 = featureAvgQuats[backQuat]; + quat1 = featureAvgQuats[backQuat + 1]; + quat2 = featureAvgQuats[backQuat + 2]; + quat3 = featureAvgQuats[backQuat + 3]; + const ebsdlib::QuatD q2(quat0, quat1, quat2, quat3); + ebsdlib::AxisAngleDType axisAngle = orientationOps[currentLaueIndex]->calculateMisorientation(q1, q2); + misorientationsBuffer[triangleIdx] = static_cast(axisAngle[3] * Constants::k_180OverPiD); + continue; + } + } + misorientationsBuffer[triangleIdx] = static_cast(std::nan("0")); + } + result = MergeResults(std::move(result), misorientations.getDataStoreRef().copyFromBuffer(start, nonstd::span(misorientationsBuffer.data(), count))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-98424, fmt::format("Error writing face misorientations at triangle {} ({} triangles).", start, count))); + } + start += count; + } + return result; +} +} // namespace + // ----------------------------------------------------------------------------- ComputeFeatureFaceMisorientation::ComputeFeatureFaceMisorientation(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, ComputeFeatureFaceMisorientationInputValues* inputValues) @@ -197,6 +366,14 @@ Result<> ComputeFeatureFaceMisorientation::operator()() return validationResult; } + const bool usesOutOfCoreStore = AnyOutOfCore({&faceLabelsArrayRef, &avgQuatsArrayRef, &featurePhasesArrayRef, &crystalStructuresArrayRef, &misorientationsArrayRef}); + const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + return ComputeFeatureMisorientationInBlocks(faceLabelsArrayRef, featurePhasesArrayRef, avgQuatsArrayRef, crystalStructuresArrayRef, misorientationsArrayRef, *m_InputValues, m_ShouldCancel); + } + ParallelDataAlgorithm parallelTask; parallelTask.setRange(0, numTriangles); parallelTask.setParallelizationEnabled(false); diff --git a/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/OrientationTupleLookup.hpp b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/OrientationTupleLookup.hpp new file mode 100644 index 0000000000..575b7810d0 --- /dev/null +++ b/src/Plugins/OrientationAnalysis/src/OrientationAnalysis/Filters/Algorithms/OrientationTupleLookup.hpp @@ -0,0 +1,210 @@ +#pragma once + +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" +#include "simplnx/DataStructure/DataPath.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" + +#include +#include + +#include +#include +#include +#include + +/** + * @namespace nx::core::orientation::detail + * @brief Provides private admitted buffers and serial tuple gathers for face-orientation algorithms. + */ +namespace nx::core::orientation::detail +{ +/** + * @struct LookupPlan + * @brief Keeps the shared reservation alive until the caller releases its lookup and face buffers. + */ +struct LookupPlan +{ + CacheMemoryBudgetManager::WorkingMemoryReservation Reservation; + usize FaceCapacity = 0; + usize PageCapacity = 0; + bool CacheWholeTables = false; +}; + +/** + * @struct LookupSizes + * @brief Names the tuple counts and payload widths used to plan both face-orientation algorithms. + */ +struct LookupSizes +{ + usize FaceCount = 0; + usize FeatureCount = 0; + usize EnsembleCount = 0; + usize OrientationComponents = 0; + usize FaceBytes = 0; +}; + +/** + * @brief Plans simultaneous face, lookup, order, and page payloads from the actual shared grant. + * @param sizes Supplies tuple counts and per-tuple payload widths. + * @param errorCode Identifies the caller's allocation-planning error. + * @return An admitted plan, or an error when one face and the minimum lookup pages do not fit. + */ +inline Result PlanLookups(const LookupSizes& sizes, int32 errorCode) +{ + constexpr usize maximumFaces = 262144; + constexpr usize maximumPageTuples = 4096; + constexpr uint64 maximumBytes = (std::numeric_limits::max)(); + const usize desiredFaces = (std::min)(sizes.FaceCount, maximumFaces); + const usize orientationBytes = sizes.OrientationComponents * sizeof(float32); + const usize featureBytes = sizeof(int32) + orientationBytes; + const usize pageBytes = featureBytes + sizeof(uint32); + const usize gatheredFaceBytes = sizes.FaceBytes + (2 * (pageBytes + sizeof(usize))); + auto& budget = CacheMemoryBudgetManager::instance(); + LookupPlan plan; + + // A complete cache is optional. Its reservation also covers the ordinary face block. + if(sizes.FeatureCount <= maximumBytes / featureBytes && sizes.EnsembleCount <= maximumBytes / sizeof(uint32)) + { + const uint64 featurePayload = static_cast(sizes.FeatureCount) * featureBytes; + const uint64 ensemblePayload = static_cast(sizes.EnsembleCount) * sizeof(uint32); + const uint64 facePayload = static_cast(desiredFaces) * sizes.FaceBytes; + if(featurePayload <= maximumBytes - ensemblePayload && featurePayload + ensemblePayload <= maximumBytes - facePayload) + { + plan.Reservation = budget.tryReserveWorkingMemoryWithoutEviction(featurePayload + ensemblePayload + facePayload); + if(plan.Reservation.sizeBytes() != 0) + { + plan.FaceCapacity = desiredFaces; + plan.CacheWholeTables = true; + Result result; + result.value() = std::move(plan); + return result; + } + } + } + + const usize desiredPages = (std::max)(usize{1}, (std::min)({maximumPageTuples, desiredFaces * 2, (std::max)(sizes.FeatureCount, sizes.EnsembleCount)})); + const uint64 desiredBytes = (static_cast(desiredFaces) * gatheredFaceBytes) + (static_cast(desiredPages) * pageBytes); + plan.Reservation = budget.reserveWorkingMemory(desiredBytes); + const uint64 grantedBytes = plan.Reservation.sizeBytes(); + const usize minimumBytes = gatheredFaceBytes + pageBytes; + if(grantedBytes < minimumBytes) + { + return MakeErrorResult(errorCode, fmt::format("Face-orientation buffers require at least {} working bytes, but the shared cache budget admitted {}. " + "Increase the cache memory budget or release other working buffers.", + minimumBytes, grantedBytes)); + } + + // Keep the ordinary face block when it fits. Smaller grants retain useful pages as well as faces. + if(grantedBytes >= static_cast(desiredFaces) * gatheredFaceBytes + pageBytes) + { + plan.FaceCapacity = desiredFaces; + } + else + { + plan.FaceCapacity = (std::max)(usize{1}, static_cast(grantedBytes / (gatheredFaceBytes + (2 * pageBytes)))); + } + plan.PageCapacity = (std::min)(desiredPages, static_cast((grantedBytes - (static_cast(plan.FaceCapacity) * gatheredFaceBytes)) / pageBytes)); + plan.Reservation.shrinkTo((static_cast(plan.FaceCapacity) * gatheredFaceBytes) + (static_cast(plan.PageCapacity) * pageBytes)); + Result result; + result.value() = std::move(plan); + return result; +} + +/** + * @brief Reads a lookup table while preserving store diagnostics and adding the selected path on failure. + * @tparam T Specifies the lookup value type. + * @param store Supplies the selected table. + * @param values Receives the admitted complete table. + * @param errorCode Identifies the caller's lookup-read error. + * @param path Identifies the selected array for diagnostics. + * @return The store result, with path context on failure. + */ +template +Result<> ReadLookupTable(const AbstractDataStore& store, nonstd::span values, int32 errorCode, const DataPath& path) +{ + auto result = store.copyIntoBuffer(0, values); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(errorCode, fmt::format("Could not read orientation lookup array '{}'. Check that its data store is available.", path.toString()))); + } + return result; +} + +/** + * @brief Gathers positive tuple IDs in page order into stable caller-owned output slots. + * @tparam T Specifies the lookup value type. + * @tparam Components Specifies the complete tuple width. + * @param store Supplies the selected table. + * @param indexes Supplies one tuple ID per destination slot. Nonpositive IDs produce zero values. + * @param order Supplies reusable sort positions, one per request. + * @param gathered Receives complete tuples in the original request order. + * @param page Supplies an admitted complete-tuple read page. + * @param shouldCancel Requests cancellation between page reads. + * @param errorCode Identifies the caller's lookup-read error. + * @param path Identifies the selected array for diagnostics. + * @param reuseOrder Reuses positions already sorted for the identical request IDs. + * @return Store warnings or an error with the selected path. Cancellation leaves unused output slots incomplete. + */ +template +Result<> GatherLookupTuples(const AbstractDataStore& store, nonstd::span indexes, nonstd::span order, nonstd::span gathered, nonstd::span page, + const std::atomic_bool& shouldCancel, int32 errorCode, const DataPath& path, bool reuseOrder = false) +{ + constexpr usize maximum = (std::numeric_limits::max)(); + if(indexes.size() > maximum / Components || gathered.size() < indexes.size() * Components || order.size() < indexes.size() || page.empty() || page.size() % Components != 0 || + store.getNumberOfComponents() != Components || store.getNumberOfTuples() > maximum / Components) + { + return MakeErrorResult( + errorCode, fmt::format("Orientation lookup array '{}' requires {} components and complete tuple buffers. Check the selected array and working buffer sizes.", path.toString(), Components)); + } + std::fill_n(gathered.begin(), indexes.size() * Components, T{}); + usize requests = 0; + for(usize slot = 0; slot < indexes.size(); ++slot) + { + if(indexes[slot] > 0) + { + if(static_cast(indexes[slot]) >= store.getNumberOfTuples()) + { + return MakeErrorResult(errorCode, fmt::format("Orientation lookup array '{}' contains {} tuples, but requested tuple {} is outside [0, {}). Check the selected feature and phase arrays.", + path.toString(), store.getNumberOfTuples(), indexes[slot], store.getNumberOfTuples())); + } + if(!reuseOrder) + { + order[requests] = slot; + } + ++requests; + } + } + if(!reuseOrder) + { + std::ranges::sort(order.first(requests), [&indexes](usize left, usize right) { return indexes[left] < indexes[right]; }); + } + const usize pageTuples = page.size() / Components; + usize loadedPage = maximum; + Result<> result; + for(usize request = 0; request < requests; ++request) + { + const usize slot = order[request]; + const usize tuple = static_cast(indexes[slot]); + const usize first = (tuple / pageTuples) * pageTuples; + if(first != loadedPage) + { + if(shouldCancel) + { + return result; + } + const usize count = (std::min)(pageTuples, store.getNumberOfTuples() - first); + result = MergeResults(std::move(result), store.copyIntoBuffer(first * Components, page.first(count * Components))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(errorCode, fmt::format("Could not read orientation lookup array '{}' at tuple {} ({} tuples). Check that its data store is available.", + path.toString(), first, count))); + } + loadedPage = first; + } + std::ranges::copy(page.subspan((tuple - first) * Components, Components), gathered.subspan(slot * Components, Components).begin()); + } + return result; +} +} // namespace nx::core::orientation::detail diff --git a/src/Plugins/OrientationAnalysis/test/AlgorithmDispatchStateTest.cpp b/src/Plugins/OrientationAnalysis/test/AlgorithmDispatchStateTest.cpp index 728272dbdd..0ddd57a61f 100644 --- a/src/Plugins/OrientationAnalysis/test/AlgorithmDispatchStateTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/AlgorithmDispatchStateTest.cpp @@ -1,5 +1,8 @@ #include "AlgorithmDispatchTestSupport.hpp" +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Core/Application.hpp" +#include "simplnx/Core/Preferences.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/ListStore.hpp" #include "simplnx/DataStructure/NeighborList.hpp" @@ -10,6 +13,10 @@ #include +#include +#include +#include +#include #include using namespace nx::core; @@ -100,6 +107,84 @@ Result<> dispatchEmptyForTest(bool& usedDirect) { return DispatchAlgorithm({}, usedDirect); } + +/** + * @class AlgorithmScopeStateGuard + * @brief Restores test state if an assertion interrupts the scope restoration test. + * + * This guard does not use the scope or preference sentinel under test. + */ +class AlgorithmScopeStateGuard +{ +public: + /** + * @brief Saves the preferences and dispatch state before the test changes them. + * @param preferences Supplies the process preferences that outlive this guard. + */ + explicit AlgorithmScopeStateGuard(Preferences& preferences) + : m_Preferences(preferences) + , m_HadMode(preferences.contains(Preferences::k_DataStorageMode_Key)) + , m_Mode(preferences.value(Preferences::k_DataStorageMode_Key)) + , m_HadThreshold(preferences.contains(Preferences::k_LargeDataSize_Key)) + , m_Threshold(preferences.value(Preferences::k_LargeDataSize_Key)) + , m_ForceInCore(ForceInCoreAlgorithm()) + , m_ForceOutOfCore(ForceOocAlgorithm()) + , m_Counts(GetAlgorithmPathExecutionCounts()) + { + } + + /** + * @brief Restores saved state without writing a preferences file. + */ + ~AlgorithmScopeStateGuard() + { + ForceInCoreAlgorithm() = m_ForceInCore; + ForceOocAlgorithm() = m_ForceOutOfCore; + SetAlgorithmPathExecutionCounts(m_Counts); + if(m_HadMode) + { + m_Preferences.setValue(Preferences::k_DataStorageMode_Key, m_Mode); + } + else + { + m_Preferences.removeValue(Preferences::k_DataStorageMode_Key); + } + if(m_HadThreshold) + { + m_Preferences.setValue(Preferences::k_LargeDataSize_Key, m_Threshold); + } + else + { + m_Preferences.removeValue(Preferences::k_LargeDataSize_Key); + } + } + + AlgorithmScopeStateGuard(const AlgorithmScopeStateGuard&) = delete; + AlgorithmScopeStateGuard(AlgorithmScopeStateGuard&&) = delete; + AlgorithmScopeStateGuard& operator=(const AlgorithmScopeStateGuard&) = delete; + AlgorithmScopeStateGuard& operator=(AlgorithmScopeStateGuard&&) = delete; + +private: + Preferences& m_Preferences; + bool m_HadMode; + nlohmann::json m_Mode; + bool m_HadThreshold; + nlohmann::json m_Threshold; + bool m_ForceInCore; + bool m_ForceOutOfCore; + AlgorithmPathExecutionCounts m_Counts; +}; + +void checkDispatchCounts(const AlgorithmPathExecutionCounts& expected) +{ + const auto actual = GetAlgorithmPathExecutionCounts(); + CHECK(actual.InCore == expected.InCore); + CHECK(actual.OutOfCore == expected.OutOfCore); + CHECK(actual.InCoreOnInMemoryStore == expected.InCoreOnInMemoryStore); + CHECK(actual.InCoreOnOutOfCoreStore == expected.InCoreOnOutOfCoreStore); + CHECK(actual.OutOfCoreOnInMemoryStore == expected.OutOfCoreOnInMemoryStore); + CHECK(actual.OutOfCoreOnOutOfCoreStore == expected.OutOfCoreOnOutOfCoreStore); +} } // namespace TEST_CASE("OrientationAnalysis::AlgorithmDispatch recognizes list-store residency", "[OrientationAnalysis][AlgorithmDispatch]") @@ -156,29 +241,51 @@ TEST_CASE("OrientationAnalysis::AlgorithmDispatch recognizes list-store residenc TEST_CASE("OrientationAnalysis::AlgorithmDispatch force state crosses the plugin boundary", "[OrientationAnalysis][AlgorithmDispatch]") { + UnitTest::LoadPlugins(); using Path = UnitTest::AlgorithmDispatchPath; - - REQUIRE_FALSE(ForceInCoreAlgorithm()); - + struct SelectionCase { - const ForceOocAlgorithmGuard forceDirect(false); - REQUIRE(UnitTest::GetAlgorithmDispatchPathFromOrientationAnalysisPlugin() == Path::Direct); - } - + bool ForceInCore = false; + bool ForceOoc = false; + Path ExpectedPath = Path::Direct; + AlgorithmPathExecutionCounts ExpectedCounts{}; + }; + const std::array cases = {{ + {.ForceInCore = false, + .ForceOoc = false, + .ExpectedPath = Path::Direct, + .ExpectedCounts = {.InCore = 1, .OutOfCore = 0, .InCoreOnInMemoryStore = 1, .InCoreOnOutOfCoreStore = 0, .OutOfCoreOnInMemoryStore = 0, .OutOfCoreOnOutOfCoreStore = 0}}, + {.ForceInCore = false, + .ForceOoc = true, + .ExpectedPath = Path::Scanline, + .ExpectedCounts = {.InCore = 0, .OutOfCore = 1, .InCoreOnInMemoryStore = 0, .InCoreOnOutOfCoreStore = 0, .OutOfCoreOnInMemoryStore = 1, .OutOfCoreOnOutOfCoreStore = 0}}, + {.ForceInCore = true, + .ForceOoc = false, + .ExpectedPath = Path::Direct, + .ExpectedCounts = {.InCore = 1, .OutOfCore = 0, .InCoreOnInMemoryStore = 1, .InCoreOnOutOfCoreStore = 0, .OutOfCoreOnInMemoryStore = 0, .OutOfCoreOnOutOfCoreStore = 0}}, + {.ForceInCore = true, + .ForceOoc = true, + .ExpectedPath = Path::Direct, + .ExpectedCounts = {.InCore = 1, .OutOfCore = 0, .InCoreOnInMemoryStore = 1, .InCoreOnOutOfCoreStore = 0, .OutOfCoreOnInMemoryStore = 0, .OutOfCoreOnOutOfCoreStore = 0}}, + }}; + const bool originalInCore = ForceInCoreAlgorithm(); + const bool originalOoc = ForceOocAlgorithm(); + const auto originalCounts = GetAlgorithmPathExecutionCounts(); + const auto restoreState = MakeScopeGuard([=]() noexcept { + ForceInCoreAlgorithm() = originalInCore; + ForceOocAlgorithm() = originalOoc; + SetAlgorithmPathExecutionCounts(originalCounts); + }); + + for(const auto& selection : cases) { - const ForceOocAlgorithmGuard forceScanline(true); - REQUIRE(UnitTest::GetAlgorithmDispatchPathFromOrientationAnalysisPlugin() == Path::Scanline); - - { - const ForceInCoreAlgorithmGuard forceDirect; - REQUIRE(UnitTest::GetAlgorithmDispatchPathFromOrientationAnalysisPlugin() == Path::Direct); - } - - REQUIRE(UnitTest::GetAlgorithmDispatchPathFromOrientationAnalysisPlugin() == Path::Scanline); + CAPTURE(selection.ForceInCore, selection.ForceOoc); + ForceInCoreAlgorithm() = selection.ForceInCore; + ForceOocAlgorithm() = selection.ForceOoc; + ResetAlgorithmPathExecutionCounts(); + REQUIRE(UnitTest::GetAlgorithmDispatchPathFromOrientationAnalysisPlugin() == selection.ExpectedPath); + checkDispatchCounts(selection.ExpectedCounts); } - - REQUIRE_FALSE(ForceOocAlgorithm()); - REQUIRE_FALSE(ForceInCoreAlgorithm()); } TEST_CASE("OrientationAnalysis::AlgorithmDispatch records the selected plugin path", "[OrientationAnalysis][AlgorithmDispatch]") @@ -289,3 +396,84 @@ TEST_CASE("OrientationAnalysis::AlgorithmTestScope runs OOC algorithm on in-memo scope.requireExpectedStore(*values); CHECK(scope.execute([] { return UnitTest::GetAlgorithmDispatchPathFromOrientationAnalysisPlugin(); }) == Path::Scanline); } + +TEST_CASE("OrientationAnalysis::AlgorithmTestScope restores state when the target throws", "[OrientationAnalysis][AlgorithmDispatch][AlgorithmTestScenario]") +{ + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool hadMode = GENERATE(false, true); + const bool hadThreshold = GENERATE(false, true); + const bool originalForceInCore = GENERATE(false, true); + const bool originalForceOutOfCore = GENERATE(false, true); + CAPTURE(scenario, hadMode, hadThreshold, originalForceInCore, originalForceOutOfCore); + + UnitTest::LoadPlugins(); + auto* preferences = Application::GetOrCreateInstance()->getPreferences(); + const AlgorithmScopeStateGuard restoreTestState(*preferences); + + if(hadMode) + { + preferences->setDataStorageMode(DataStorageMode::Adaptive); + } + else + { + preferences->removeValue(Preferences::k_DataStorageMode_Key); + } + if(hadThreshold) + { + preferences->setValue(Preferences::k_LargeDataSize_Key, std::numeric_limits::max()); + } + else + { + preferences->removeValue(Preferences::k_LargeDataSize_Key); + } + const auto originalMode = preferences->value(Preferences::k_DataStorageMode_Key); + const auto originalThreshold = preferences->value(Preferences::k_LargeDataSize_Key); + const AlgorithmPathExecutionCounts originalCounts{11, 13, 17, 19, 23, 29}; + ForceInCoreAlgorithm() = originalForceInCore; + ForceOocAlgorithm() = originalForceOutOfCore; + SetAlgorithmPathExecutionCounts(originalCounts); + + { + UnitTest::AlgorithmTestScope scope(scenario); + const bool expectsOutOfCore = scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore; + bool caughtExpectedException = false; + // The outer handler keeps the original exception alive for the identity check. + try + { + throw std::runtime_error("public algorithm scope target failure"); + } catch(const std::runtime_error& expected) + { + try + { + scope.execute([&] { + CHECK(ForceInCoreAlgorithm() == !expectsOutOfCore); + CHECK(ForceOocAlgorithm() == expectsOutOfCore); + checkDispatchCounts({}); + REQUIRE(preferences->dataStorageMode() == DataStorageMode::ForceInCore); + REQUIRE(preferences->value(Preferences::k_LargeDataSize_Key).type() == originalThreshold.type()); + REQUIRE(preferences->value(Preferences::k_LargeDataSize_Key) == originalThreshold); + // The synthetic event tests scope cleanup; it does not represent a filter execution. + RecordAlgorithmPathExecution(expectsOutOfCore ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, false); + throw; + }); + } catch(const std::runtime_error& actual) + { + caughtExpectedException = true; + CHECK(&actual == &expected); + CHECK(std::string(actual.what()) == "public algorithm scope target failure"); + } + } + REQUIRE(caughtExpectedException); + CHECK(ForceInCoreAlgorithm() == originalForceInCore); + CHECK(ForceOocAlgorithm() == originalForceOutOfCore); + checkDispatchCounts(originalCounts); + CHECK(preferences->dataStorageMode() == DataStorageMode::ForceInCore); + } + + CHECK(preferences->contains(Preferences::k_DataStorageMode_Key) == hadMode); + CHECK(preferences->value(Preferences::k_DataStorageMode_Key).type() == originalMode.type()); + CHECK(preferences->value(Preferences::k_DataStorageMode_Key) == originalMode); + CHECK(preferences->contains(Preferences::k_LargeDataSize_Key) == hadThreshold); + CHECK(preferences->value(Preferences::k_LargeDataSize_Key).type() == originalThreshold.type()); + CHECK(preferences->value(Preferences::k_LargeDataSize_Key) == originalThreshold); +} diff --git a/src/Plugins/OrientationAnalysis/test/CMakeLists.txt b/src/Plugins/OrientationAnalysis/test/CMakeLists.txt index 3360d17f3d..a66f5d7b14 100644 --- a/src/Plugins/OrientationAnalysis/test/CMakeLists.txt +++ b/src/Plugins/OrientationAnalysis/test/CMakeLists.txt @@ -439,3 +439,13 @@ if(TARGET ImageProcessing) endif() create_pipeline_tests(PLUGIN_NAME ${PLUGIN_NAME} PIPELINE_LIST ${PREBUILT_PIPELINE_NAMES}) + +target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${simplnx_SOURCE_DIR}/test/ProfileIsolationContract.cpp") + +add_test(NAME "${PLUGIN_NAME}_test_profile_isolation" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -DPROFILE_RUNNER_ONLY=TRUE + -P "${simplnx_SOURCE_DIR}/test/VerifyTestProfileLifecycle.cmake") +set_tests_properties("${PLUGIN_NAME}_test_profile_isolation" PROPERTIES RUN_SERIAL TRUE TIMEOUT 60) diff --git a/src/Plugins/OrientationAnalysis/test/ComputeAvgOrientationsTest.cpp b/src/Plugins/OrientationAnalysis/test/ComputeAvgOrientationsTest.cpp index e07c56404e..ca33c8f625 100644 --- a/src/Plugins/OrientationAnalysis/test/ComputeAvgOrientationsTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ComputeAvgOrientationsTest.cpp @@ -760,6 +760,9 @@ TEST_CASE("OrientationAnalysis::ComputeAvgOrientations: Rodrigues Cubic Symmetry TEST_CASE("OrientationAnalysis::ComputeAvgOrientations: Rodrigues Voxel Ordering Independence", "[OrientationAnalysis][ComputeAvgOrientationsFilter]") { UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); const std::vector quatIdentity = {0.0f, 0.0f, 0.0f, 1.0f}; const std::vector quatRz90 = {0.0f, 0.0f, k_Rz90_z, k_Rz90_z}; @@ -778,6 +781,15 @@ TEST_CASE("OrientationAnalysis::ComputeAvgOrientations: Rodrigues Voxel Ordering const std::vector crystalStructures = {k_Unknown, k_Triclinic}; DataStructure dataStructure = BuildDataStructure(2, 2, featureIds, phases, quats, crystalStructures); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_FeatureIdsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_FeatureIdsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_PhasesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_PhasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_QuatsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_QuatsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_CrystalStructuresPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_CrystalStructuresPath)); + ComputeAvgOrientationsFilter filter; Arguments args; SetInputArgs(args); @@ -795,13 +807,17 @@ TEST_CASE("OrientationAnalysis::ComputeAvgOrientations: Rodrigues Voxel Ordering auto preflightResult = filter.preflight(dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_FeatureDataPath.createChildPath(k_AvgQuatsName))); const auto& avgQuats = dataStructure.getDataRefAs(k_FeatureDataPath.createChildPath(k_AvgQuatsName)); + scope.requireExpectedStore(avgQuats); CheckQuat(avgQuats, 1, 0.0f, 0.0f, k_Rz45_z, k_Rz45_w, 1.0e-6f); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_FeatureDataPath.createChildPath(k_AvgEulerName))); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_FeatureDataPath.createChildPath(k_AvgEulerName))); + UnitTest::CheckArraysInheritTupleDims(dataStructure); } } diff --git a/src/Plugins/OrientationAnalysis/test/ComputeFaceIPFColoringTest.cpp b/src/Plugins/OrientationAnalysis/test/ComputeFaceIPFColoringTest.cpp index be427b9117..e470b445d4 100644 --- a/src/Plugins/OrientationAnalysis/test/ComputeFaceIPFColoringTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ComputeFaceIPFColoringTest.cpp @@ -12,7 +12,9 @@ #include "simplnx/Parameters/ChoicesParameter.hpp" #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" +#include "simplnx/UnitTest/AlgorithmTestScope.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include @@ -44,6 +46,56 @@ DataPath faceDataGroup = triangleDataContainerPath.createChildPath(nx::core::Con DataPath faceLabels = faceDataGroup.createChildPath(nx::core::Constants::k_FaceLabels); DataPath faceNormals = faceDataGroup.createChildPath(nx::core::Constants::k_FaceNormals); DataPath faceAreas = faceDataGroup.createChildPath(nx::core::Constants::k_FaceAreas); + +const DataPath k_BoundsFaceLabelsPath({"Bounds Data", "FaceLabels"}); +const DataPath k_BoundsFaceNormalsPath({"Bounds Data", "FaceNormals"}); +const DataPath k_BoundsEulerAnglesPath({"Bounds Data", "FeatureEulerAngles"}); +const DataPath k_BoundsFeaturePhasesPath({"Bounds Data", "FeaturePhases"}); +const DataPath k_BoundsCrystalStructuresPath({"Bounds Data", "CrystalStructures"}); + +Arguments BuildPhaseBoundsFixture(DataStructure& dataStructure) +{ + auto* dataGroupPtr = DataGroup::Create(dataStructure, "Bounds Data"); + REQUIRE(dataGroupPtr != nullptr); + + auto faceLabelsStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsFaceLabelsPath, {1}, {2}); + auto* faceLabelsArrayPtr = Int32Array::Create(dataStructure, k_BoundsFaceLabelsPath.getTargetName(), faceLabelsStore, dataGroupPtr->getId()); + REQUIRE(faceLabelsArrayPtr != nullptr); + (*faceLabelsStore)[0] = 1; + (*faceLabelsStore)[1] = 0; + + auto faceNormalsStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsFaceNormalsPath, {1}, {3}); + auto* faceNormalsArrayPtr = Float64Array::Create(dataStructure, k_BoundsFaceNormalsPath.getTargetName(), faceNormalsStore, dataGroupPtr->getId()); + REQUIRE(faceNormalsArrayPtr != nullptr); + (*faceNormalsStore)[2] = 1.0; + + auto eulerAnglesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsEulerAnglesPath, {2}, {3}); + auto* eulerAnglesArrayPtr = Float32Array::Create(dataStructure, k_BoundsEulerAnglesPath.getTargetName(), eulerAnglesStore, dataGroupPtr->getId()); + REQUIRE(eulerAnglesArrayPtr != nullptr); + + auto featurePhasesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsFeaturePhasesPath, {2}, {1}); + auto* featurePhasesArrayPtr = Int32Array::Create(dataStructure, k_BoundsFeaturePhasesPath.getTargetName(), featurePhasesStore, dataGroupPtr->getId()); + REQUIRE(featurePhasesArrayPtr != nullptr); + (*featurePhasesStore)[0] = 0; + (*featurePhasesStore)[1] = 2; + + auto crystalStructuresStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsCrystalStructuresPath, {2}, {1}); + auto* crystalStructuresArrayPtr = UInt32Array::Create(dataStructure, k_BoundsCrystalStructuresPath.getTargetName(), crystalStructuresStore, dataGroupPtr->getId()); + REQUIRE(crystalStructuresArrayPtr != nullptr); + (*crystalStructuresStore)[0] = ebsdlib::CrystalStructure::UnknownCrystalStructure; + (*crystalStructuresStore)[1] = ebsdlib::CrystalStructure::Cubic_High; + + ComputeFaceIPFColoringFilter filter; + Arguments args = filter.getDefaultArguments(); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SurfaceMeshFaceLabelsArrayPath_Key, std::make_any(k_BoundsFaceLabelsPath)); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SurfaceMeshFaceNormalsArrayPath_Key, std::make_any(k_BoundsFaceNormalsPath)); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FeatureEulerAnglesArrayPath_Key, std::make_any(k_BoundsEulerAnglesPath)); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FeaturePhasesArrayPath_Key, std::make_any(k_BoundsFeaturePhasesPath)); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_CrystalStructuresArrayPath_Key, std::make_any(k_BoundsCrystalStructuresPath)); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FirstFaceIPFColorsArrayName_Key, std::make_any("First Bounds Colors")); + args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SecondFaceIPFColorsArrayName_Key, std::make_any("Second Bounds Colors")); + return args; +} } // namespace // ============================================================================= @@ -110,6 +162,10 @@ TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: Class 1 Oracle - m { UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + // EbsdLib crystal-structure enum values used as ops[] indices. constexpr uint32 k_HexHigh = 0; // Hexagonal_High (6/mmm) constexpr uint32 k_CubicHigh = 1; // Cubic_High (m-3m) @@ -165,9 +221,25 @@ TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: Class 1 Oracle - m auto preflightResult = filter.preflight(dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); - auto executeResult = filter.execute(dataStructure, args); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceLabelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(faceLabelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceNormalsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(faceNormalsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(eulerAnglesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(eulerAnglesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(phasesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(phasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(crystalStructuresPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(crystalStructuresPath)); + + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceLabelsPath.replaceName(::k_FirstNXFaceIPFColors))); + scope.requireExpectedStore(dataStructure.getDataRefAs(faceLabelsPath.replaceName(::k_FirstNXFaceIPFColors))); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceLabelsPath.replaceName(::k_SecondNXFaceIPFColors))); + scope.requireExpectedStore(dataStructure.getDataRefAs(faceLabelsPath.replaceName(::k_SecondNXFaceIPFColors))); + const auto& first = dataStructure.getDataRefAs(faceLabelsPath.replaceName(::k_FirstNXFaceIPFColors)); const auto& second = dataStructure.getDataRefAs(faceLabelsPath.replaceName(::k_SecondNXFaceIPFColors)); @@ -196,63 +268,59 @@ TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: Class 1 Oracle - m UnitTest::CheckArraysInheritTupleDims(dataStructure); } +TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: working headroom applies only to OOC computation", "[OrientationAnalysis][ComputeFaceIPFColoringFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + UnitTest::AlgorithmTestScope scope(scenario); + DataStructure dataStructure; + Arguments args = BuildPhaseBoundsFixture(dataStructure); + Int32Array* phasesPtr = nullptr; + REQUIRE_NOTHROW(phasesPtr = &dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + (*phasesPtr)[1] = 1; + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + auto pressure = budget.reserveWorkingMemory(budget.maximumWorkingMemoryBytes()); + REQUIRE(pressure.sizeBytes() == budget.maximumWorkingMemoryBytes()); + ComputeFaceIPFColoringFilter filter; + auto result = scope.executeFilter(filter, dataStructure, args); + if(scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore) + { + SIMPLNX_RESULT_REQUIRE_INVALID(result.result); + REQUIRE(result.result.errors()[0].code == -24359); + } + else + { + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + const UInt8Array* firstPtr = nullptr; + REQUIRE_NOTHROW(firstPtr = &dataStructure.getDataRefAs(k_BoundsFaceLabelsPath.replaceName("First Bounds Colors"))); + const std::array expected = {255, 0, 0}; + for(usize component = 0; component < 3; ++component) + { + REQUIRE((*firstPtr)[component] == expected.at(component)); + } + } + CHECK(budget.reservedWorkingMemoryBytes() == pressure.sizeBytes()); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: Phase Index Bounds", "[OrientationAnalysis][ComputeFaceIPFColoringFilter]") { UnitTest::LoadPlugins(); const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); DataStructure dataStructure; - auto* dataGroupPtr = DataGroup::Create(dataStructure, "Bounds Data"); - REQUIRE(dataGroupPtr != nullptr); - - const DataPath faceLabelsPath({"Bounds Data", "FaceLabels"}); - const DataPath faceNormalsPath({"Bounds Data", "FaceNormals"}); - const DataPath eulerAnglesPath({"Bounds Data", "FeatureEulerAngles"}); - const DataPath boundsFeaturePhasesPath({"Bounds Data", "FeaturePhases"}); - const DataPath crystalStructuresPath({"Bounds Data", "CrystalStructures"}); - - auto faceLabelsStore = DataStoreUtilities::CreateDataStore(dataStructure, faceLabelsPath, {1}, {2}); - auto* faceLabelsArrayPtr = Int32Array::Create(dataStructure, faceLabelsPath.getTargetName(), faceLabelsStore, dataGroupPtr->getId()); - REQUIRE(faceLabelsArrayPtr != nullptr); - (*faceLabelsStore)[0] = 1; - (*faceLabelsStore)[1] = 0; - - auto faceNormalsStore = DataStoreUtilities::CreateDataStore(dataStructure, faceNormalsPath, {1}, {3}); - auto* faceNormalsArrayPtr = Float64Array::Create(dataStructure, faceNormalsPath.getTargetName(), faceNormalsStore, dataGroupPtr->getId()); - REQUIRE(faceNormalsArrayPtr != nullptr); - (*faceNormalsStore)[2] = 1.0; - - auto eulerAnglesStore = DataStoreUtilities::CreateDataStore(dataStructure, eulerAnglesPath, {2}, {3}); - auto* eulerAnglesArrayPtr = Float32Array::Create(dataStructure, eulerAnglesPath.getTargetName(), eulerAnglesStore, dataGroupPtr->getId()); - REQUIRE(eulerAnglesArrayPtr != nullptr); - - auto featurePhasesStore = DataStoreUtilities::CreateDataStore(dataStructure, boundsFeaturePhasesPath, {2}, {1}); - auto* featurePhasesArrayPtr = Int32Array::Create(dataStructure, boundsFeaturePhasesPath.getTargetName(), featurePhasesStore, dataGroupPtr->getId()); - REQUIRE(featurePhasesArrayPtr != nullptr); - (*featurePhasesStore)[0] = 0; - (*featurePhasesStore)[1] = 2; - - auto crystalStructuresStore = DataStoreUtilities::CreateDataStore(dataStructure, crystalStructuresPath, {2}, {1}); - auto* crystalStructuresArrayPtr = UInt32Array::Create(dataStructure, crystalStructuresPath.getTargetName(), crystalStructuresStore, dataGroupPtr->getId()); - REQUIRE(crystalStructuresArrayPtr != nullptr); - (*crystalStructuresStore)[0] = ebsdlib::CrystalStructure::UnknownCrystalStructure; - (*crystalStructuresStore)[1] = ebsdlib::CrystalStructure::Cubic_High; + Arguments args = BuildPhaseBoundsFixture(dataStructure); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); if(Application::Instance()->getIOManager("HDF5-OOC") != nullptr) { - REQUIRE(featurePhasesStore->getDataFormat() == "HDF5-OOC"); - REQUIRE(faceLabelsStore->getDataFormat() == "HDF5-OOC"); + REQUIRE(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath).getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath).getDataStoreRef().getDataFormat() == "HDF5-OOC"); } ComputeFaceIPFColoringFilter filter; - Arguments args = filter.getDefaultArguments(); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SurfaceMeshFaceLabelsArrayPath_Key, std::make_any(faceLabelsPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SurfaceMeshFaceNormalsArrayPath_Key, std::make_any(faceNormalsPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FeatureEulerAnglesArrayPath_Key, std::make_any(eulerAnglesPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FeaturePhasesArrayPath_Key, std::make_any(boundsFeaturePhasesPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_CrystalStructuresArrayPath_Key, std::make_any(crystalStructuresPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FirstFaceIPFColorsArrayName_Key, std::make_any("First Bounds Colors")); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SecondFaceIPFColorsArrayName_Key, std::make_any("Second Bounds Colors")); SECTION("First Feature Phase returns an error") { @@ -263,132 +331,51 @@ TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: Phase Index Bounds SECTION("Second Feature Phase returns an error") { - (*faceLabelsStore)[0] = 0; - (*faceLabelsStore)[1] = 1; + auto& faceLabelsStore = dataStructure.getDataRefAs(k_BoundsFaceLabelsPath).getDataStoreRef(); + faceLabelsStore[0] = 0; + faceLabelsStore[1] = 1; auto executeResult = filter.execute(dataStructure, args); SIMPLNX_RESULT_REQUIRE_INVALID(executeResult.result); REQUIRE(executeResult.result.errors()[0].code == -24341); } - SECTION("Unreferenced Feature Phase is ignored") - { - (*faceLabelsStore)[0] = 0; - (*faceLabelsStore)[1] = 0; - auto executeResult = filter.execute(dataStructure, args); - SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); - } - UnitTest::CheckArraysInheritTupleDims(dataStructure); } -TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: genuine HDF5 65536-block tail oracle", "[OrientationAnalysis][ComputeFaceIPFColoringFilter][.OocStoreContract]") +TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: Unreferenced Feature Phase resident scenarios", "[OrientationAnalysis][ComputeFaceIPFColoringFilter]") { UnitTest::LoadPlugins(); - REQUIRE(Application::Instance()->getIOManager("HDF5-OOC") != nullptr); - - constexpr usize k_BlockTuples = 65536; - constexpr usize k_FaceTuples = k_BlockTuples + 1; - constexpr usize k_LastFullBlockTuple = k_BlockTuples - 1; - constexpr usize k_TailTuple = k_BlockTuples; - const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); - - const DataPath dataPath({"Boundary Data"}); - const DataPath faceLabelsPath = dataPath.createChildPath("FaceLabels"); - const DataPath faceNormalsPath = dataPath.createChildPath("FaceNormals"); - const DataPath eulerAnglesPath = dataPath.createChildPath("FeatureEulerAngles"); - const DataPath phasesPath = dataPath.createChildPath("FeaturePhases"); - const DataPath crystalStructuresPath = dataPath.createChildPath("CrystalStructures"); - const DataPath firstColorsPath = dataPath.createChildPath("First Boundary Colors"); - const DataPath secondColorsPath = dataPath.createChildPath("Second Boundary Colors"); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); DataStructure dataStructure; - auto* dataGroup = DataGroup::Create(dataStructure, dataPath.getTargetName()); - REQUIRE(dataGroup != nullptr); - - auto faceLabelsStore = DataStoreUtilities::CreateDataStore(dataStructure, faceLabelsPath, {k_FaceTuples}, {2}); - auto faceNormalsStore = DataStoreUtilities::CreateDataStore(dataStructure, faceNormalsPath, {k_FaceTuples}, {3}); - auto eulerAnglesStore = DataStoreUtilities::CreateDataStore(dataStructure, eulerAnglesPath, {3}, {3}); - auto phasesStore = DataStoreUtilities::CreateDataStore(dataStructure, phasesPath, {3}, {1}); - auto crystalStructuresStore = DataStoreUtilities::CreateDataStore(dataStructure, crystalStructuresPath, {3}, {1}); - - auto* faceLabelsArray = Int32Array::Create(dataStructure, faceLabelsPath.getTargetName(), faceLabelsStore, dataGroup->getId()); - auto* faceNormalsArray = Float64Array::Create(dataStructure, faceNormalsPath.getTargetName(), faceNormalsStore, dataGroup->getId()); - auto* eulerAnglesArray = Float32Array::Create(dataStructure, eulerAnglesPath.getTargetName(), eulerAnglesStore, dataGroup->getId()); - auto* phasesArray = Int32Array::Create(dataStructure, phasesPath.getTargetName(), phasesStore, dataGroup->getId()); - auto* crystalStructuresArray = UInt32Array::Create(dataStructure, crystalStructuresPath.getTargetName(), crystalStructuresStore, dataGroup->getId()); - REQUIRE(faceLabelsArray != nullptr); - REQUIRE(faceNormalsArray != nullptr); - REQUIRE(eulerAnglesArray != nullptr); - REQUIRE(phasesArray != nullptr); - REQUIRE(crystalStructuresArray != nullptr); - - std::vector faceLabelValues(k_FaceTuples * 2, -1); - std::vector faceNormalValues(k_FaceTuples * 3, 0.0); - faceLabelValues[k_LastFullBlockTuple * 2] = 1; - faceNormalValues[k_LastFullBlockTuple * 3] = -1.0; - faceLabelValues[k_TailTuple * 2 + 1] = 2; - faceNormalValues[k_TailTuple * 3] = -1.0; - - const std::array eulerAngleValues = {0.0F, 0.0F, 0.0F, nx::core::Constants::k_PiOver2F, 0.0F, 0.0F, nx::core::Constants::k_PiOver3F, 0.0F, 0.0F}; - const std::array phaseValues = {0, 1, 2}; - const std::array crystalStructureValues = {ebsdlib::CrystalStructure::UnknownCrystalStructure, ebsdlib::CrystalStructure::Cubic_High, ebsdlib::CrystalStructure::Hexagonal_High}; - - auto faceLabelsWriteResult = faceLabelsArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(faceLabelValues.data(), faceLabelValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(faceLabelsWriteResult); - auto faceNormalsWriteResult = faceNormalsArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(faceNormalValues.data(), faceNormalValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(faceNormalsWriteResult); - auto eulerAnglesWriteResult = eulerAnglesArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(eulerAngleValues.data(), eulerAngleValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(eulerAnglesWriteResult); - auto phasesWriteResult = phasesArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(phaseValues.data(), phaseValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(phasesWriteResult); - auto crystalStructuresWriteResult = crystalStructuresArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(crystalStructureValues.data(), crystalStructureValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(crystalStructuresWriteResult); - - REQUIRE(faceLabelsArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(faceNormalsArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(eulerAnglesArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(phasesArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(crystalStructuresArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + Arguments args = BuildPhaseBoundsFixture(dataStructure); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); + auto& faceLabelsStore = dataStructure.getDataRefAs(k_BoundsFaceLabelsPath).getDataStoreRef(); + faceLabelsStore[0] = 0; + faceLabelsStore[1] = 0; + + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceNormalsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFaceNormalsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsEulerAnglesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsEulerAnglesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsCrystalStructuresPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsCrystalStructuresPath)); ComputeFaceIPFColoringFilter filter; - Arguments args = filter.getDefaultArguments(); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SurfaceMeshFaceLabelsArrayPath_Key, std::make_any(faceLabelsPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SurfaceMeshFaceNormalsArrayPath_Key, std::make_any(faceNormalsPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FeatureEulerAnglesArrayPath_Key, std::make_any(eulerAnglesPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FeaturePhasesArrayPath_Key, std::make_any(phasesPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_CrystalStructuresArrayPath_Key, std::make_any(crystalStructuresPath)); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_FirstFaceIPFColorsArrayName_Key, std::make_any(firstColorsPath.getTargetName())); - args.insertOrAssign(ComputeFaceIPFColoringFilter::k_SecondFaceIPFColorsArrayName_Key, std::make_any(secondColorsPath.getTargetName())); - - auto preflightResult = filter.preflight(dataStructure, args); - SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); - REQUIRE(executeResult.result.warnings().empty()); - - REQUIRE_NOTHROW(dataStructure.getDataRefAs(firstColorsPath)); - REQUIRE_NOTHROW(dataStructure.getDataRefAs(secondColorsPath)); - const auto& firstColors = dataStructure.getDataRefAs(firstColorsPath); - const auto& secondColors = dataStructure.getDataRefAs(secondColorsPath); - REQUIRE(firstColors.getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(secondColors.getDataStoreRef().getDataFormat() == "HDF5-OOC"); - - const usize lastFullBlockOffset = k_LastFullBlockTuple * 3; - REQUIRE(firstColors[lastFullBlockOffset] == 255); - REQUIRE(firstColors[lastFullBlockOffset + 1] == 0); - REQUIRE(firstColors[lastFullBlockOffset + 2] == 0); - REQUIRE(secondColors[lastFullBlockOffset] == 0); - REQUIRE(secondColors[lastFullBlockOffset + 1] == 0); - REQUIRE(secondColors[lastFullBlockOffset + 2] == 0); - - const usize tailOffset = k_TailTuple * 3; - REQUIRE(firstColors[tailOffset] == 0); - REQUIRE(firstColors[tailOffset + 1] == 0); - REQUIRE(firstColors[tailOffset + 2] == 0); - REQUIRE(secondColors[tailOffset] == 0); - REQUIRE(secondColors[tailOffset + 1] == 255); - REQUIRE(secondColors[tailOffset + 2] == 0); + + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath.replaceName("First Bounds Colors"))); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath.replaceName("First Bounds Colors"))); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath.replaceName("Second Bounds Colors"))); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath.replaceName("Second Bounds Colors"))); UnitTest::CheckArraysInheritTupleDims(dataStructure); } @@ -443,11 +430,26 @@ TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: ColorKey choice re { UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_TestFilesDir, "6_6_Small_IN100_GBCD.tar.gz", "6_6_Small_IN100_GBCD"); auto baseDataFilePath = fs::path(fmt::format("{}/6_6_Small_IN100_GBCD/6_6_Small_IN100_GBCD.dream3d", unit_test::k_TestFilesDir)); DataStructure dataStructure = UnitTest::LoadDataStructure(baseDataFilePath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceLabels)); + scope.requireExpectedStore(dataStructure.getDataRefAs(faceLabels)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceNormals)); + scope.requireExpectedStore(dataStructure.getDataRefAs(faceNormals)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(avgEulerAnglesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(avgEulerAnglesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(featurePhasesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(featurePhasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(crystalStructurePath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(crystalStructurePath)); + // Run the filter once per kind, writing into uniquely-named output arrays. auto runWithKind = [&](ChoicesParameter::ValueType kindIndex, const std::string& firstName, const std::string& secondName) { ComputeFaceIPFColoringFilter filter; @@ -463,8 +465,15 @@ TEST_CASE("OrientationAnalysis::ComputeFaceIPFColoringFilter: ColorKey choice re auto preflightResult = filter.preflight(dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + const DataPath firstColorsPath = faceLabels.replaceName(firstName); + const DataPath secondColorsPath = faceLabels.replaceName(secondName); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(firstColorsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(firstColorsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(secondColorsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(secondColorsPath)); }; runWithKind(0, "FirstIPF_TSL", "SecondIPF_TSL"); diff --git a/src/Plugins/OrientationAnalysis/test/ComputeFeatureFaceMisorientationTest.cpp b/src/Plugins/OrientationAnalysis/test/ComputeFeatureFaceMisorientationTest.cpp index 623d4cb6bf..d7acf6bfe7 100644 --- a/src/Plugins/OrientationAnalysis/test/ComputeFeatureFaceMisorientationTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ComputeFeatureFaceMisorientationTest.cpp @@ -9,7 +9,9 @@ #include "simplnx/DataStructure/DataGroup.hpp" #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" +#include "simplnx/UnitTest/AlgorithmTestScope.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include @@ -439,6 +441,54 @@ DataStructure CreateTestDataStructure() return dataStructure; } } // namespace curated + +const DataPath k_BoundsFaceLabelsPath({"Bounds Data", "FaceLabels"}); +const DataPath k_BoundsAvgQuatsPath({"Bounds Data", "AvgQuats"}); +const DataPath k_BoundsFeaturePhasesPath({"Bounds Data", "FeaturePhases"}); +const DataPath k_BoundsCrystalStructuresPath({"Bounds Data", "CrystalStructures"}); +const DataPath k_BoundsMisorientationsPath = k_BoundsFaceLabelsPath.replaceName("Bounds Misorientations"); + +Arguments BuildPhaseBoundsFixture(DataStructure& dataStructure) +{ + auto* dataGroupPtr = DataGroup::Create(dataStructure, "Bounds Data"); + REQUIRE(dataGroupPtr != nullptr); + + auto faceLabelsStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsFaceLabelsPath, {1}, {2}); + auto* faceLabelsArrayPtr = Int32Array::Create(dataStructure, k_BoundsFaceLabelsPath.getTargetName(), faceLabelsStore, dataGroupPtr->getId()); + REQUIRE(faceLabelsArrayPtr != nullptr); + (*faceLabelsStore)[0] = 1; + (*faceLabelsStore)[1] = 2; + + auto avgQuatsStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsAvgQuatsPath, {3}, {4}); + auto* avgQuatsArrayPtr = Float32Array::Create(dataStructure, k_BoundsAvgQuatsPath.getTargetName(), avgQuatsStore, dataGroupPtr->getId()); + REQUIRE(avgQuatsArrayPtr != nullptr); + for(usize featureIdx = 0; featureIdx < 3; featureIdx++) + { + avgQuatsStore->setComponent(featureIdx, 3, 1.0F); + } + + auto featurePhasesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsFeaturePhasesPath, {3}, {1}); + auto* featurePhasesArrayPtr = Int32Array::Create(dataStructure, k_BoundsFeaturePhasesPath.getTargetName(), featurePhasesStore, dataGroupPtr->getId()); + REQUIRE(featurePhasesArrayPtr != nullptr); + (*featurePhasesStore)[0] = 0; + (*featurePhasesStore)[1] = 2; + (*featurePhasesStore)[2] = 2; + + auto crystalStructuresStore = DataStoreUtilities::CreateDataStore(dataStructure, k_BoundsCrystalStructuresPath, {2}, {1}); + auto* crystalStructuresArrayPtr = UInt32Array::Create(dataStructure, k_BoundsCrystalStructuresPath.getTargetName(), crystalStructuresStore, dataGroupPtr->getId()); + REQUIRE(crystalStructuresArrayPtr != nullptr); + (*crystalStructuresStore)[0] = ebsdlib::CrystalStructure::UnknownCrystalStructure; + (*crystalStructuresStore)[1] = ebsdlib::CrystalStructure::Cubic_High; + + ComputeFeatureFaceMisorientationFilter filter; + Arguments args = filter.getDefaultArguments(); + args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_SurfaceMeshFaceLabelsArrayPath_Key, std::make_any(k_BoundsFaceLabelsPath)); + args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_AvgQuatsArrayPath_Key, std::make_any(k_BoundsAvgQuatsPath)); + args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_FeaturePhasesArrayPath_Key, std::make_any(k_BoundsFeaturePhasesPath)); + args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_CrystalStructuresArrayPath_Key, std::make_any(k_BoundsCrystalStructuresPath)); + args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_MisorientationArrayName_Key, std::make_any(k_BoundsMisorientationsPath.getTargetName())); + return args; +} } // namespace TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Curated Data", "[OrientationAnalysis][ComputeFeatureFaceMisorientationFilter]") @@ -447,6 +497,10 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Curated UnitTest::LoadPlugins(); auto filterList = app->getFilterList(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + DataStructure dataStructure = curated::CreateTestDataStructure(); // Convert the EulerAngles to radians @@ -485,6 +539,15 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Curated SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); } + REQUIRE_NOTHROW(dataStructure.getDataRefAs(curated::k_FaceLabelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(curated::k_FaceLabelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(curated::k_AvgQuatsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(curated::k_AvgQuatsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(curated::k_FeaturePhasesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(curated::k_FeaturePhasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(curated::k_CrystalStructurePath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(curated::k_CrystalStructurePath)); + // ComputeFeatureFaceMisorientationFilter { // Instantiate the filter, and an Arguments Object @@ -503,10 +566,13 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Curated SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); // Execute the filter and check the result - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); } + REQUIRE_NOTHROW(dataStructure.getDataRefAs(curated::k_FaceDataPath.createChildPath(::k_NXFaceMisorientationColors))); + scope.requireExpectedStore(dataStructure.getDataRefAs(curated::k_FaceDataPath.createChildPath(::k_NXFaceMisorientationColors))); + // Validate Computed Misorientations const auto& faceMisorientations = dataStructure.getDataRefAs(curated::k_FaceDataPath.createChildPath(::k_NXFaceMisorientationColors)); @@ -564,60 +630,56 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Curated UnitTest::CheckArraysInheritTupleDims(dataStructure); } -TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Phase Index Bounds", "[OrientationAnalysis][ComputeFeatureFaceMisorientationFilter]") +TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: working headroom applies only to OOC computation", "[OrientationAnalysis][ComputeFeatureFaceMisorientationFilter]") { UnitTest::LoadPlugins(); - - const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + UnitTest::AlgorithmTestScope scope(scenario); DataStructure dataStructure; - auto* dataGroupPtr = DataGroup::Create(dataStructure, "Bounds Data"); - REQUIRE(dataGroupPtr != nullptr); - - const DataPath faceLabelsPath({"Bounds Data", "FaceLabels"}); - const DataPath avgQuatsPath({"Bounds Data", "AvgQuats"}); - const DataPath featurePhasesPath({"Bounds Data", "FeaturePhases"}); - const DataPath crystalStructuresPath({"Bounds Data", "CrystalStructures"}); - - auto faceLabelsStore = DataStoreUtilities::CreateDataStore(dataStructure, faceLabelsPath, {1}, {2}); - auto* faceLabelsArrayPtr = Int32Array::Create(dataStructure, faceLabelsPath.getTargetName(), faceLabelsStore, dataGroupPtr->getId()); - REQUIRE(faceLabelsArrayPtr != nullptr); - (*faceLabelsStore)[0] = 1; - (*faceLabelsStore)[1] = 2; - - auto avgQuatsStore = DataStoreUtilities::CreateDataStore(dataStructure, avgQuatsPath, {3}, {4}); - auto* avgQuatsArrayPtr = Float32Array::Create(dataStructure, avgQuatsPath.getTargetName(), avgQuatsStore, dataGroupPtr->getId()); - REQUIRE(avgQuatsArrayPtr != nullptr); - for(usize featureIdx = 0; featureIdx < 3; featureIdx++) + Arguments args = BuildPhaseBoundsFixture(dataStructure); + Int32Array* phasesPtr = nullptr; + REQUIRE_NOTHROW(phasesPtr = &dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + (*phasesPtr)[1] = 1; + (*phasesPtr)[2] = 1; + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + auto pressure = budget.reserveWorkingMemory(budget.maximumWorkingMemoryBytes()); + REQUIRE(pressure.sizeBytes() == budget.maximumWorkingMemoryBytes()); + ComputeFeatureFaceMisorientationFilter filter; + auto result = scope.executeFilter(filter, dataStructure, args); + if(scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore) { - avgQuatsStore->setComponent(featureIdx, 3, 1.0F); + SIMPLNX_RESULT_REQUIRE_INVALID(result.result); + REQUIRE(result.result.errors()[0].code == -98425); } + else + { + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + const Float32Array* anglesPtr = nullptr; + REQUIRE_NOTHROW(anglesPtr = &dataStructure.getDataRefAs(k_BoundsMisorientationsPath)); + REQUIRE((*anglesPtr)[0] == 0.0F); + } + CHECK(budget.reservedWorkingMemoryBytes() == pressure.sizeBytes()); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} - auto featurePhasesStore = DataStoreUtilities::CreateDataStore(dataStructure, featurePhasesPath, {3}, {1}); - auto* featurePhasesArrayPtr = Int32Array::Create(dataStructure, featurePhasesPath.getTargetName(), featurePhasesStore, dataGroupPtr->getId()); - REQUIRE(featurePhasesArrayPtr != nullptr); - (*featurePhasesStore)[0] = 0; - (*featurePhasesStore)[1] = 2; - (*featurePhasesStore)[2] = 2; +TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Phase Index Bounds", "[OrientationAnalysis][ComputeFeatureFaceMisorientationFilter]") +{ + UnitTest::LoadPlugins(); - auto crystalStructuresStore = DataStoreUtilities::CreateDataStore(dataStructure, crystalStructuresPath, {2}, {1}); - auto* crystalStructuresArrayPtr = UInt32Array::Create(dataStructure, crystalStructuresPath.getTargetName(), crystalStructuresStore, dataGroupPtr->getId()); - REQUIRE(crystalStructuresArrayPtr != nullptr); - (*crystalStructuresStore)[0] = ebsdlib::CrystalStructure::UnknownCrystalStructure; - (*crystalStructuresStore)[1] = ebsdlib::CrystalStructure::Cubic_High; + const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); + DataStructure dataStructure; + Arguments args = BuildPhaseBoundsFixture(dataStructure); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); if(Application::Instance()->getIOManager("HDF5-OOC") != nullptr) { - REQUIRE(featurePhasesStore->getDataFormat() == "HDF5-OOC"); - REQUIRE(faceLabelsStore->getDataFormat() == "HDF5-OOC"); + REQUIRE(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath).getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath).getDataStoreRef().getDataFormat() == "HDF5-OOC"); } ComputeFeatureFaceMisorientationFilter filter; - Arguments args = filter.getDefaultArguments(); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_SurfaceMeshFaceLabelsArrayPath_Key, std::make_any(faceLabelsPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_AvgQuatsArrayPath_Key, std::make_any(avgQuatsPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_FeaturePhasesArrayPath_Key, std::make_any(featurePhasesPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_CrystalStructuresArrayPath_Key, std::make_any(crystalStructuresPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_MisorientationArrayName_Key, std::make_any("Bounds Misorientations")); SECTION("Participating Phase returns an error") { @@ -626,108 +688,40 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Phase In REQUIRE(executeResult.result.errors()[0].code == -98412); } - SECTION("Mismatched Phase is ignored") - { - (*featurePhasesStore)[2] = 1; - auto executeResult = filter.execute(dataStructure, args); - SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); - REQUIRE_NOTHROW(dataStructure.getDataRefAs(faceLabelsPath.replaceName("Bounds Misorientations"))); - const auto& misorientationsArrayRef = dataStructure.getDataRefAs(faceLabelsPath.replaceName("Bounds Misorientations")); - REQUIRE(std::isnan(misorientationsArrayRef[0])); - } - UnitTest::CheckArraysInheritTupleDims(dataStructure); } -TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: genuine HDF5 65536-block tail oracle", "[OrientationAnalysis][ComputeFeatureFaceMisorientationFilter][.OocStoreContract]") +TEST_CASE("OrientationAnalysis::ComputeFeatureFaceMisorientationFilter: Mismatched Phase resident scenarios", "[OrientationAnalysis][ComputeFeatureFaceMisorientationFilter]") { UnitTest::LoadPlugins(); - REQUIRE(Application::Instance()->getIOManager("HDF5-OOC") != nullptr); - constexpr usize k_BlockTuples = 65536; - constexpr usize k_FaceTuples = k_BlockTuples + 1; - constexpr usize k_LastFullBlockTuple = k_BlockTuples - 1; - constexpr usize k_TailTuple = k_BlockTuples; - constexpr float32 k_FifteenDegrees = 15.0F; - constexpr float32 k_ThirtyDegrees = 30.0F; - - const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); - - const DataPath dataPath({"Boundary Data"}); - const DataPath faceLabelsPath = dataPath.createChildPath("FaceLabels"); - const DataPath avgQuatsPath = dataPath.createChildPath("AvgQuats"); - const DataPath featurePhasesPath = dataPath.createChildPath("FeaturePhases"); - const DataPath crystalStructuresPath = dataPath.createChildPath("CrystalStructures"); - const DataPath misorientationsPath = dataPath.createChildPath("Boundary Misorientations"); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); DataStructure dataStructure; - auto* dataGroup = DataGroup::Create(dataStructure, dataPath.getTargetName()); - REQUIRE(dataGroup != nullptr); - - auto faceLabelsStore = DataStoreUtilities::CreateDataStore(dataStructure, faceLabelsPath, {k_FaceTuples}, {2}); - auto avgQuatsStore = DataStoreUtilities::CreateDataStore(dataStructure, avgQuatsPath, {4}, {4}); - auto featurePhasesStore = DataStoreUtilities::CreateDataStore(dataStructure, featurePhasesPath, {4}, {1}); - auto crystalStructuresStore = DataStoreUtilities::CreateDataStore(dataStructure, crystalStructuresPath, {2}, {1}); - - auto* faceLabelsArray = Int32Array::Create(dataStructure, faceLabelsPath.getTargetName(), faceLabelsStore, dataGroup->getId()); - auto* avgQuatsArray = Float32Array::Create(dataStructure, avgQuatsPath.getTargetName(), avgQuatsStore, dataGroup->getId()); - auto* featurePhasesArray = Int32Array::Create(dataStructure, featurePhasesPath.getTargetName(), featurePhasesStore, dataGroup->getId()); - auto* crystalStructuresArray = UInt32Array::Create(dataStructure, crystalStructuresPath.getTargetName(), crystalStructuresStore, dataGroup->getId()); - REQUIRE(faceLabelsArray != nullptr); - REQUIRE(avgQuatsArray != nullptr); - REQUIRE(featurePhasesArray != nullptr); - REQUIRE(crystalStructuresArray != nullptr); - - std::vector faceLabelValues(k_FaceTuples * 2, -1); - faceLabelValues[k_LastFullBlockTuple * 2] = 1; - faceLabelValues[k_LastFullBlockTuple * 2 + 1] = 2; - faceLabelValues[k_TailTuple * 2] = 1; - faceLabelValues[k_TailTuple * 2 + 1] = 3; - - const float32 sinSevenPointFiveDegrees = std::sin(7.5F * Constants::k_PiOver180F); - const float32 cosSevenPointFiveDegrees = std::cos(7.5F * Constants::k_PiOver180F); - const float32 sinFifteenDegrees = std::sin(k_FifteenDegrees * Constants::k_PiOver180F); - const float32 cosFifteenDegrees = std::cos(k_FifteenDegrees * Constants::k_PiOver180F); - const std::array avgQuatValues = { - 0.0F, 0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 0.0F, 1.0F, 0.0F, 0.0F, sinSevenPointFiveDegrees, cosSevenPointFiveDegrees, 0.0F, 0.0F, sinFifteenDegrees, cosFifteenDegrees}; - const std::array phaseValues = {0, 1, 1, 1}; - const std::array crystalStructureValues = {ebsdlib::CrystalStructure::UnknownCrystalStructure, ebsdlib::CrystalStructure::Cubic_High}; - - auto faceLabelsWriteResult = faceLabelsArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(faceLabelValues.data(), faceLabelValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(faceLabelsWriteResult); - auto avgQuatsWriteResult = avgQuatsArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(avgQuatValues.data(), avgQuatValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(avgQuatsWriteResult); - auto featurePhasesWriteResult = featurePhasesArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(phaseValues.data(), phaseValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(featurePhasesWriteResult); - auto crystalStructuresWriteResult = crystalStructuresArray->getDataStoreRef().copyFromBuffer(0, nonstd::span(crystalStructureValues.data(), crystalStructureValues.size())); - SIMPLNX_RESULT_REQUIRE_VALID(crystalStructuresWriteResult); - - REQUIRE(faceLabelsArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(avgQuatsArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(featurePhasesArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(crystalStructuresArray->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + Arguments args = BuildPhaseBoundsFixture(dataStructure); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + auto& featurePhasesStore = dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath).getDataStoreRef(); + featurePhasesStore[2] = 1; + + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFaceLabelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsAvgQuatsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsAvgQuatsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsFeaturePhasesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsCrystalStructuresPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsCrystalStructuresPath)); ComputeFeatureFaceMisorientationFilter filter; - Arguments args = filter.getDefaultArguments(); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_SurfaceMeshFaceLabelsArrayPath_Key, std::make_any(faceLabelsPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_AvgQuatsArrayPath_Key, std::make_any(avgQuatsPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_FeaturePhasesArrayPath_Key, std::make_any(featurePhasesPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_CrystalStructuresArrayPath_Key, std::make_any(crystalStructuresPath)); - args.insertOrAssign(ComputeFeatureFaceMisorientationFilter::k_MisorientationArrayName_Key, std::make_any(misorientationsPath.getTargetName())); - - auto preflightResult = filter.preflight(dataStructure, args); - SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); - REQUIRE(executeResult.result.warnings().empty()); - - REQUIRE_NOTHROW(dataStructure.getDataRefAs(misorientationsPath)); - const auto& misorientations = dataStructure.getDataRefAs(misorientationsPath); - REQUIRE(misorientations.getDataStoreRef().getDataFormat() == "HDF5-OOC"); - REQUIRE(misorientations.getTupleShape() == ShapeType{k_FaceTuples}); - REQUIRE(misorientations.getComponentShape() == ShapeType{1}); - REQUIRE(misorientations[k_LastFullBlockTuple] == Approx(k_FifteenDegrees).margin(0.000012F)); - REQUIRE(misorientations[k_TailTuple] == Approx(k_ThirtyDegrees).margin(0.000012F)); + + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_BoundsMisorientationsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_BoundsMisorientationsPath)); + const auto& misorientationsArrayRef = dataStructure.getDataRefAs(k_BoundsMisorientationsPath); + REQUIRE(std::isnan(misorientationsArrayRef[0])); UnitTest::CheckArraysInheritTupleDims(dataStructure); } diff --git a/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborCAxisMisalignmentsTest.cpp b/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborCAxisMisalignmentsTest.cpp index 3650ed6c1d..43220bfe52 100644 --- a/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborCAxisMisalignmentsTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborCAxisMisalignmentsTest.cpp @@ -131,6 +131,30 @@ void SetAvgQuat(FixtureData& td, usize featureIdx, const std::array& (*td.avgQuats)[featureIdx * 4 + 3] = q[3]; } +void PopulateMixedHexNeighbors(FixtureData& fixture) +{ + REQUIRE(fixture.totalFeatures == 5); + REQUIRE(fixture.featurePhases != nullptr); + REQUIRE(fixture.avgQuats != nullptr); + REQUIRE(fixture.neighborList != nullptr); + REQUIRE(fixture.crystalStructures != nullptr); + + (*fixture.crystalStructures)[1] = static_cast(ebsdlib::CrystalStructure::Hexagonal_High); + (*fixture.crystalStructures)[2] = static_cast(ebsdlib::CrystalStructure::Cubic_High); + (*fixture.featurePhases)[0] = 0; + (*fixture.featurePhases)[1] = 1; + (*fixture.featurePhases)[2] = 1; + (*fixture.featurePhases)[3] = 1; + (*fixture.featurePhases)[4] = 2; + SetAvgQuat(fixture, 1, QuatFromPhiDeg(0.0f)); + SetAvgQuat(fixture, 2, QuatFromPhiDeg(5.0f)); + SetAvgQuat(fixture, 3, QuatFromPhiDeg(10.0f)); + SetAvgQuat(fixture, 4, QuatFromPhiDeg(20.0f)); + + // The cubic neighbor separates two valid hexagonal neighbors in the output list. + fixture.neighborList->setList(1, std::vector{2, 4, 3}); +} + Arguments BuildArgs(bool findAvgMisals) { Arguments args; @@ -451,6 +475,102 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborCAxisMisalignmentsFilter: REQUIRE(avg[1] == Approx(7.5f).margin(1e-3f)); } +TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborCAxisMisalignmentsFilter: Averages Disabled - Resident Lists", "[OrientationAnalysis][ComputeFeatureNeighborCAxisMisalignmentsFilter]") +{ + UnitTest::LoadPlugins(); + const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceInCore, 1); + + AnalyticalFixtures::FixtureData fixture = AnalyticalFixtures::CreateScaffold(/*nX=*/1, /*nY=*/1, /*nZ=*/1, /*numFeatures=*/5, /*numCrystalStructures=*/3); + AnalyticalFixtures::PopulateMixedHexNeighbors(fixture); + + REQUIRE(fixture.featurePhases->getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(fixture.avgQuats->getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(fixture.crystalStructures->getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(fixture.neighborList->getIListStore() != nullptr); + REQUIRE_FALSE(fixture.neighborList->getIListStore()->isOutOfCore()); + const auto inputLists = fixture.neighborList->getVectors(); + REQUIRE(inputLists.size() == 5); + + ComputeFeatureNeighborCAxisMisalignmentsFilter filter; + const Arguments args = AnalyticalFixtures::BuildArgs(/*findAvgMisals=*/false); + auto preflightResult = filter.preflight(fixture.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = filter.execute(fixture.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + REQUIRE(std::any_of(executeResult.result.warnings().begin(), executeResult.result.warnings().end(), [](const Warning& warning) { return warning.code == -1563; })); + + const auto& misalignmentList = AnalyticalFixtures::GetOutputMisalignmentList(fixture.ds); + REQUIRE(misalignmentList.getIListStore() != nullptr); + REQUIRE_FALSE(misalignmentList.getIListStore()->isOutOfCore()); + REQUIRE(misalignmentList.getNumberOfLists() == 5); + const std::array expectedListLengths = {0, 3, 0, 0, 0}; + for(usize featureIdx = 0; featureIdx < expectedListLengths.size(); ++featureIdx) + { + CAPTURE(featureIdx); + REQUIRE(misalignmentList.at(featureIdx).size() == expectedListLengths[featureIdx]); + } + const auto focalMisalignments = misalignmentList.at(1); + REQUIRE(focalMisalignments[0] == Approx(5.0f).margin(1e-3f)); + REQUIRE(std::isnan(focalMisalignments[1])); + REQUIRE(focalMisalignments[2] == Approx(10.0f).margin(1e-3f)); + REQUIRE(fixture.ds.getData(AnalyticalFixtures::k_FeatureDataPath.createChildPath(AnalyticalFixtures::k_AvgCAxisMisalignmentsOutName)) == nullptr); + REQUIRE(fixture.neighborList->getVectors() == inputLists); + + UnitTest::CheckArraysInheritTupleDims(fixture.ds); +} + +TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborCAxisMisalignmentsFilter: Averages Disabled - HDF5 Lists", + "[OrientationAnalysis][ComputeFeatureNeighborCAxisMisalignmentsFilter][.OocStoreContract]") +{ + UnitTest::LoadPlugins(); + REQUIRE(Application::Instance()->getIOManager("HDF5-OOC") != nullptr); + const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); + + AnalyticalFixtures::FixtureData fixture = AnalyticalFixtures::CreateScaffold(/*nX=*/1, /*nY=*/1, /*nZ=*/1, /*numFeatures=*/5, /*numCrystalStructures=*/3); + REQUIRE(fixture.neighborList != nullptr); + const DataPath neighborListPath = AnalyticalFixtures::k_FeatureDataPath.createChildPath(AnalyticalFixtures::k_NeighborListName); + auto neighborListStore = DataStoreUtilities::CreateListStore(fixture.ds, neighborListPath, {5}, IDataAction::Mode::Execute, "HDF5-OOC"); + REQUIRE(neighborListStore != nullptr); + // The scaffold creates a resident list store, so replace the store before list population. + fixture.neighborList->setStore(neighborListStore); + AnalyticalFixtures::PopulateMixedHexNeighbors(fixture); + + REQUIRE(fixture.featurePhases->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(fixture.avgQuats->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(fixture.crystalStructures->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(fixture.neighborList->getIListStore() != nullptr); + REQUIRE(fixture.neighborList->getIListStore()->isOutOfCore()); + const auto inputLists = fixture.neighborList->getVectors(); + REQUIRE(inputLists.size() == 5); + + ComputeFeatureNeighborCAxisMisalignmentsFilter filter; + const Arguments args = AnalyticalFixtures::BuildArgs(/*findAvgMisals=*/false); + auto preflightResult = filter.preflight(fixture.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = filter.execute(fixture.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + REQUIRE(std::any_of(executeResult.result.warnings().begin(), executeResult.result.warnings().end(), [](const Warning& warning) { return warning.code == -1563; })); + + const auto& misalignmentList = AnalyticalFixtures::GetOutputMisalignmentList(fixture.ds); + REQUIRE(misalignmentList.getIListStore() != nullptr); + REQUIRE(misalignmentList.getIListStore()->isOutOfCore()); + REQUIRE(misalignmentList.getNumberOfLists() == 5); + const std::array expectedListLengths = {0, 3, 0, 0, 0}; + for(usize featureIdx = 0; featureIdx < expectedListLengths.size(); ++featureIdx) + { + CAPTURE(featureIdx); + REQUIRE(misalignmentList.at(featureIdx).size() == expectedListLengths[featureIdx]); + } + const auto focalMisalignments = misalignmentList.at(1); + REQUIRE(focalMisalignments[0] == Approx(5.0f).margin(1e-3f)); + REQUIRE(std::isnan(focalMisalignments[1])); + REQUIRE(focalMisalignments[2] == Approx(10.0f).margin(1e-3f)); + REQUIRE(fixture.ds.getData(AnalyticalFixtures::k_FeatureDataPath.createChildPath(AnalyticalFixtures::k_AvgCAxisMisalignmentsOutName)) == nullptr); + REQUIRE(fixture.neighborList->getVectors() == inputLists); + + UnitTest::CheckArraysInheritTupleDims(fixture.ds); +} + TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborCAxisMisalignmentsFilter: Class 4 - Invariants", "[OrientationAnalysis][ComputeFeatureNeighborCAxisMisalignmentsFilter]") { UnitTest::LoadPlugins(); diff --git a/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborMisorientationsTest.cpp b/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborMisorientationsTest.cpp index 992423a1ef..21f39a8365 100644 --- a/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborMisorientationsTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ComputeFeatureNeighborMisorientationsTest.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace fs = std::filesystem; using namespace nx::core; @@ -120,6 +121,30 @@ void SetAvgQuat(FixtureData& td, usize featureIdx, const std::array& (*td.avgQuats)[featureIdx * 4 + 3] = q[3]; } +void PopulateAveragesDisabledFixture(FixtureData& td) +{ + REQUIRE(td.featurePhases != nullptr); + REQUIRE(td.avgQuats != nullptr); + REQUIRE(td.neighborList != nullptr); + REQUIRE(td.crystalStructures != nullptr); + REQUIRE(td.featurePhases->getDataStore() != nullptr); + REQUIRE(td.avgQuats->getDataStore() != nullptr); + REQUIRE(td.neighborList->getIListStore() != nullptr); + REQUIRE(td.crystalStructures->getDataStore() != nullptr); + + const std::array featurePhases = {0, 1, 1, 1, 2}; + const std::array phi1Degrees = {0.0f, 0.0f, 5.0f, 10.0f, 99.0f}; + for(usize featureIdx = 0; featureIdx < featurePhases.size(); featureIdx++) + { + (*td.featurePhases)[featureIdx] = featurePhases[featureIdx]; + SetAvgQuat(td, featureIdx, QuatFromPhi1Deg(phi1Degrees[featureIdx])); + td.neighborList->setList(featureIdx, std::vector{}); + } + (*td.crystalStructures)[1] = ebsdlib::CrystalStructure::Cubic_High; + (*td.crystalStructures)[2] = ebsdlib::CrystalStructure::Hexagonal_High; + td.neighborList->setList(1, std::vector{2, 4, 3}); +} + Arguments BuildArgs(bool computeAvgMisors) { Arguments args; @@ -407,6 +432,94 @@ TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborMisorientationsFilter: Cla REQUIRE(avg[1] == Approx(expectedAvg).margin(1e-4f)); } +TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborMisorientationsFilter: Averages Disabled - Resident Lists", "[OrientationAnalysis][ComputeFeatureNeighborMisorientationsFilter]") +{ + UnitTest::LoadPlugins(); + const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceInCore, 1); + AnalyticalFixtures::FixtureData td = AnalyticalFixtures::CreateScaffold(/*numFeatures=*/5, /*numCrystalStructures=*/3); + AnalyticalFixtures::PopulateAveragesDisabledFixture(td); + + REQUIRE(td.featurePhases->getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(td.avgQuats->getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(td.crystalStructures->getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE_FALSE(td.neighborList->getIListStore()->isOutOfCore()); + const std::array, 5> inputNeighborLists = {td.neighborList->at(0), td.neighborList->at(1), td.neighborList->at(2), td.neighborList->at(3), td.neighborList->at(4)}; + + ComputeFeatureNeighborMisorientationsFilter filter; + const Arguments args = AnalyticalFixtures::BuildArgs(/*computeAvgMisors=*/false); + auto preflightResult = filter.preflight(td.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = filter.execute(td.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + const auto& misoList = AnalyticalFixtures::GetOutputMisorientationList(td.ds); + REQUIRE(misoList.getIListStore() != nullptr); + REQUIRE_FALSE(misoList.getIListStore()->isOutOfCore()); + REQUIRE(misoList.getNumberOfTuples() == 5); + const std::array expectedListSizes = {0, 3, 0, 0, 0}; + for(usize featureIdx = 0; featureIdx < expectedListSizes.size(); featureIdx++) + { + CAPTURE(featureIdx); + REQUIRE(misoList.at(featureIdx).size() == expectedListSizes[featureIdx]); + REQUIRE(td.neighborList->at(featureIdx) == inputNeighborLists[featureIdx]); + } + const auto feature1List = misoList.at(1); + REQUIRE(feature1List[0] == Approx(5.0f).margin(1e-3f)); + REQUIRE(std::isnan(feature1List[1])); + REQUIRE(feature1List[2] == Approx(10.0f).margin(1e-3f)); + REQUIRE(td.ds.getData(AnalyticalFixtures::k_FeatureDataPath.createChildPath(AnalyticalFixtures::k_AvgMisorientationsOutName)) == nullptr); + + UnitTest::CheckArraysInheritTupleDims(td.ds); +} + +TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborMisorientationsFilter: Averages Disabled - HDF5 Lists", "[OrientationAnalysis][ComputeFeatureNeighborMisorientationsFilter][.OocStoreContract]") +{ + UnitTest::LoadPlugins(); + REQUIRE(Application::Instance()->getIOManager("HDF5-OOC") != nullptr); + const UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceOutOfCore, 1); + AnalyticalFixtures::FixtureData td = AnalyticalFixtures::CreateScaffold(/*numFeatures=*/5, /*numCrystalStructures=*/3); + + // The scaffold creates a resident list. Replace its store before filling the fixture. + const DataPath neighborListPath = AnalyticalFixtures::k_FeatureDataPath.createChildPath(AnalyticalFixtures::k_NeighborListName); + const auto neighborListStore = DataStoreUtilities::CreateListStore(td.ds, neighborListPath, {5}, IDataAction::Mode::Execute, "HDF5-OOC"); + REQUIRE(neighborListStore != nullptr); + REQUIRE(td.neighborList != nullptr); + td.neighborList->setStore(neighborListStore); + AnalyticalFixtures::PopulateAveragesDisabledFixture(td); + + REQUIRE(td.featurePhases->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(td.avgQuats->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(td.crystalStructures->getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(td.neighborList->getIListStore()->isOutOfCore()); + const std::array, 5> inputNeighborLists = {td.neighborList->at(0), td.neighborList->at(1), td.neighborList->at(2), td.neighborList->at(3), td.neighborList->at(4)}; + + ComputeFeatureNeighborMisorientationsFilter filter; + const Arguments args = AnalyticalFixtures::BuildArgs(/*computeAvgMisors=*/false); + auto preflightResult = filter.preflight(td.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = filter.execute(td.ds, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + const auto& misoList = AnalyticalFixtures::GetOutputMisorientationList(td.ds); + REQUIRE(misoList.getIListStore() != nullptr); + REQUIRE(misoList.getIListStore()->isOutOfCore()); + REQUIRE(misoList.getNumberOfTuples() == 5); + const std::array expectedListSizes = {0, 3, 0, 0, 0}; + for(usize featureIdx = 0; featureIdx < expectedListSizes.size(); featureIdx++) + { + CAPTURE(featureIdx); + REQUIRE(misoList.at(featureIdx).size() == expectedListSizes[featureIdx]); + REQUIRE(td.neighborList->at(featureIdx) == inputNeighborLists[featureIdx]); + } + const auto feature1List = misoList.at(1); + REQUIRE(feature1List[0] == Approx(5.0f).margin(1e-3f)); + REQUIRE(std::isnan(feature1List[1])); + REQUIRE(feature1List[2] == Approx(10.0f).margin(1e-3f)); + REQUIRE(td.ds.getData(AnalyticalFixtures::k_FeatureDataPath.createChildPath(AnalyticalFixtures::k_AvgMisorientationsOutName)) == nullptr); + + UnitTest::CheckArraysInheritTupleDims(td.ds); +} + TEST_CASE("OrientationAnalysis::ComputeFeatureNeighborMisorientationsFilter: genuine HDF5 1MiB-chunk tail oracle", "[OrientationAnalysis][ComputeFeatureNeighborMisorientationsFilter][.OocStoreContract]") { diff --git a/src/Plugins/OrientationAnalysis/test/OrientationAnalysis_test_main.cpp b/src/Plugins/OrientationAnalysis/test/OrientationAnalysis_test_main.cpp index 48425c5326..6b322c3f17 100644 --- a/src/Plugins/OrientationAnalysis/test/OrientationAnalysis_test_main.cpp +++ b/src/Plugins/OrientationAnalysis/test/OrientationAnalysis_test_main.cpp @@ -1,4 +1,11 @@ // Catch2 recommends placing these lines by themselves in a translation unit // which will help reduce unnecessary recompilations of the expensive Catch main -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_RUNNER #include + +#include "simplnx/UnitTest/OwnedCatchRunner.hpp" + +int main(int argc, char* argv[]) +{ + return nx::core::UnitTest::RunOwnedCatchSession(argc, argv); +} diff --git a/src/Plugins/OrientationAnalysis/test/ReadH5OinaDataTest.cpp b/src/Plugins/OrientationAnalysis/test/ReadH5OinaDataTest.cpp index 7809edea1d..200765e34c 100644 --- a/src/Plugins/OrientationAnalysis/test/ReadH5OinaDataTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/ReadH5OinaDataTest.cpp @@ -65,6 +65,7 @@ #include "OrientationAnalysis/OrientationAnalysis_test_dirs.hpp" #include "OrientationAnalysis/Parameters/OEMEbsdScanSelectionParameter.h" +#include "simplnx/Common/Types.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" @@ -1327,11 +1328,13 @@ TEST_CASE("OrientationAnalysis::ReadH5OinaDataFilter: genuine HDF5 batch-tail an scan.bands.assign(k_ScanTuples, scanIdx == 0 ? 3 : 5); scan.error.assign(k_ScanTuples, scanIdx == 0 ? 0 : 2); scan.mad.assign(k_ScanTuples, scanIdx == 0 ? 0.125F : 1.125F); - scan.x.assign(k_ScanTuples, scanIdx == 0 ? 0.5F : 1.5F); + scan.x.resize(k_ScanTuples); scan.y.assign(k_ScanTuples, scanIdx == 0 ? 0.25F : 1.25F); scan.euler.resize(k_ScanTuples * 3); + // Each scan is one complete row with unit X spacing and a constant Y. for(usize cellIdx = 0; cellIdx < k_ScanTuples; cellIdx++) { + scan.x[cellIdx] = static_cast(cellIdx) + (scanIdx == 0 ? 0.5F : 1.5F); scan.euler[cellIdx * 3] = scanIdx == 0 ? 0.25F : 0.75F; scan.euler[cellIdx * 3 + 1] = scanIdx == 0 ? 0.5F : 1.0F; scan.euler[cellIdx * 3 + 2] = scanIdx == 0 ? 0.1F : 0.22F; @@ -1342,8 +1345,6 @@ TEST_CASE("OrientationAnalysis::ReadH5OinaDataFilter: genuine HDF5 batch-tail an scan.bands.back() = scanIdx == 0 ? 4 : 6; scan.error.back() = scanIdx == 0 ? 1 : 3; scan.mad.back() = scanIdx == 0 ? 0.25F : 1.25F; - scan.x.back() = scanIdx == 0 ? 0.75F : 1.75F; - scan.y.back() = scanIdx == 0 ? 0.5F : 1.5F; scan.euler.back() = scanIdx == 0 ? 0.34F : 0.35F; } const auto inputFile = WriteH5OinaFixture("read_h5oina_vv_ooc_batch_tail.h5oina", scans); @@ -1372,8 +1373,8 @@ TEST_CASE("OrientationAnalysis::ReadH5OinaDataFilter: genuine HDF5 batch-tail an checkScalar(ebsdlib::H5OINA::Bands, std::array{3, 4, 5, 6}); checkScalar(ebsdlib::H5OINA::Error, std::array{0, 1, 2, 3}); checkScalar(ebsdlib::H5OINA::MeanAngularDeviation, std::array{0.125F, 0.25F, 1.125F, 1.25F}); - checkScalar(ebsdlib::H5OINA::X, std::array{0.5F, 0.75F, 1.5F, 1.75F}); - checkScalar(ebsdlib::H5OINA::Y, std::array{0.25F, 0.5F, 1.25F, 1.5F}); + checkScalar(ebsdlib::H5OINA::X, std::array{65535.5F, 65536.5F, 1.5F, 65537.5F}); + checkScalar(ebsdlib::H5OINA::Y, std::array{0.25F, 0.25F, 1.25F, 1.25F}); const auto eulerPath = k_CellAMPath.createChildPath(ebsdlib::H5OINA::Euler); REQUIRE_NOTHROW(dataStructure.getDataRefAs(eulerPath)); diff --git a/src/Plugins/OrientationAnalysis/test/WriteINLFileTest.cpp b/src/Plugins/OrientationAnalysis/test/WriteINLFileTest.cpp index bac6cce61e..a999007341 100644 --- a/src/Plugins/OrientationAnalysis/test/WriteINLFileTest.cpp +++ b/src/Plugins/OrientationAnalysis/test/WriteINLFileTest.cpp @@ -4,6 +4,7 @@ #include "OrientationAnalysis/OrientationAnalysis_test_dirs.hpp" #include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Uuid.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" @@ -25,10 +26,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -47,47 +46,179 @@ const std::string k_NumFeatures = "NumFeatures"; const fs::path k_ExemplarFilePath = fs::path(fmt::format("{}/INL_writer/INLWriterExemplar.inl", unit_test::k_TestFilesDir)); const fs::path k_WrittenFilePath = fs::path(fmt::format("{}/WriteINLFile.inl", unit_test::k_BinaryTestOutputDir)); -void CompareResults() +/** + * @brief Compares all INL lines after the variable metadata. + * @param exemplarPath Archive file with two initial metadata lines. + * @param generatedPath Generated file with one initial metadata line. + * @return Empty string on equality, or the first open, read, length, or content error. + */ +std::string FindINLFileDifference(const fs::path& exemplarPath, const fs::path& generatedPath) { - REQUIRE(fs::exists(k_WrittenFilePath)); - REQUIRE(fs::exists(k_ExemplarFilePath)); - std::ifstream exemplarFile(k_ExemplarFilePath, std::ios_base::in | std::ios_base::binary); + std::ifstream exemplarFile(exemplarPath, std::ios::binary); if(!exemplarFile.is_open()) { - throw std::runtime_error(fmt::format("{} must be stream readable!", k_ExemplarFilePath.string())); + return fmt::format("Cannot open exemplar INL file '{}'.", exemplarPath.string()); } - // Ignore the timestamp and version header lines. - exemplarFile.ignore(std::numeric_limits::max(), '\n'); - exemplarFile.ignore(std::numeric_limits::max(), '\n'); - std::ifstream generatedFile(k_WrittenFilePath, std::ios_base::in | std::ios_base::binary); - if(!exemplarFile.is_open()) + std::ifstream generatedFile(generatedPath, std::ios::binary); + if(!generatedFile.is_open()) { - throw std::runtime_error(fmt::format("{} must be stream readable!", k_WrittenFilePath.string())); + return fmt::format("Cannot open generated INL file '{}'.", generatedPath.string()); } - // Ignore the generated version line. - generatedFile.ignore(std::numeric_limits::max(), '\n'); + const auto readLine = [](std::ifstream& file, std::string& line) { + const bool hasLine = static_cast(std::getline(file, line)); + // Only a CR before the LF delimiter belongs to the line ending. + if(hasLine && !file.eof() && !line.empty() && line.back() == '\r') + { + line.pop_back(); + } + return hasLine; + }; - std::string exemplar = ""; - std::string generated = ""; - for(usize i = 0; i < 1024; i++) + std::string exemplar; + std::string generated; + // The archive has version and timestamp lines. The generated file has only a version line. + for(usize headerLine = 1; headerLine <= 2; ++headerLine) + { + if(!readLine(exemplarFile, exemplar)) + { + return fmt::format("Cannot read metadata line {} from exemplar INL file '{}'.", headerLine, exemplarPath.string()); + } + } + if(!readLine(generatedFile, generated)) { - if(exemplarFile.eof()) + return fmt::format("Cannot read metadata line 1 from generated INL file '{}'.", generatedPath.string()); + } + + for(usize comparedLine = 1;; ++comparedLine) + { + const bool hasExemplarLine = readLine(exemplarFile, exemplar); + const bool hasGeneratedLine = readLine(generatedFile, generated); + if(exemplarFile.bad() || (exemplarFile.fail() && !exemplarFile.eof())) + { + return fmt::format("Cannot read compared line {} from exemplar INL file '{}'.", comparedLine, exemplarPath.string()); + } + if(generatedFile.bad() || (generatedFile.fail() && !generatedFile.eof())) + { + return fmt::format("Cannot read compared line {} from generated INL file '{}'.", comparedLine, generatedPath.string()); + } + if(hasExemplarLine != hasGeneratedLine) { - break; + return fmt::format("{} generated content at compared line {} (exemplar line {}, generated line {}).", hasExemplarLine ? "Missing" : "Extra", comparedLine, comparedLine + 2, comparedLine + 1); } + if(!hasExemplarLine) + { + return {}; + } + if(exemplar != generated) + { + return fmt::format("Content differs at compared line {} (exemplar line {}, generated line {}): '{}' != '{}'.", comparedLine, comparedLine + 2, comparedLine + 1, exemplar, generated); + } + } +} +} // namespace + +TEST_CASE("OrientationAnalysis::WriteINLFileFilter: Complete file comparison", "[OrientationAnalysis][WriteINLFileFilter]") +{ + const fs::path fixtureDir = fs::temp_directory_path() / ("nx_inl_comparison_" + Uuid::GenerateV4().str()); + const fs::path exemplarPath = fixtureDir / "exemplar.inl"; + const fs::path generatedPath = fixtureDir / "generated.inl"; + REQUIRE(fs::create_directory(fixtureDir)); + auto fixtureGuard = MakeScopeGuard([&]() noexcept { + std::error_code errorCode; + fs::remove(exemplarPath, errorCode); + fs::remove(generatedPath, errorCode); + fs::remove(fixtureDir, errorCode); + }); + + const std::string exemplarNewline = GENERATE("\n", "\r\n"); + const std::string generatedNewline = GENERATE("\n", "\r\n"); + CAPTURE(exemplarNewline, generatedNewline); + std::string exemplarText = "# Archived version" + exemplarNewline + "# Archive timestamp" + exemplarNewline; + std::string generatedText = "# Generated version" + generatedNewline; + for(usize record = 1; record <= 1024; ++record) + { + const std::string text = fmt::format("record {}", record); + exemplarText += text + exemplarNewline; + generatedText += text + generatedNewline; + } + exemplarText += "last record" + exemplarNewline; + std::string expectedDifference; + bool writeGeneratedFile = true; - std::getline(exemplarFile, exemplar, '\n'); - std::getline(generatedFile, generated, '\n'); + SECTION("Equal complete content with different metadata") + { + generatedText += "last record" + generatedNewline; + } + SECTION("Equal final record without a line ending") + { + generatedText += "last record"; + } + SECTION("Missing generated tail after record 1024") + { + expectedDifference = "Missing generated content at compared line 1025 "; + } + SECTION("Extra generated tail after record 1024") + { + generatedText += "last record" + generatedNewline + "extra record" + generatedNewline; + expectedDifference = "Extra generated content at compared line 1026 "; + } + SECTION("Different generated tail after record 1024") + { + generatedText += "last record changed" + generatedNewline; + expectedDifference = "Content differs at compared line 1025 "; + } + SECTION("Extra blank generated tail") + { + generatedText += "last record" + generatedNewline + generatedNewline; + expectedDifference = "Extra generated content at compared line 1026 "; + } + SECTION("Record whitespace remains significant") + { + generatedText += "last record " + generatedNewline; + expectedDifference = "Content differs at compared line 1025 "; + } + SECTION("Embedded carriage return remains significant") + { + generatedText += "last record\runexpected" + generatedNewline; + expectedDifference = "Content differs at compared line 1025 "; + } + SECTION("Unterminated carriage return remains significant") + { + generatedText += "last record\r"; + expectedDifference = "Content differs at compared line 1025 "; + } + SECTION("Generated file cannot be opened") + { + writeGeneratedFile = false; + expectedDifference = fmt::format("Cannot open generated INL file '{}'.", generatedPath.string()); + } - std::string exem = exemplar.substr(0, exemplar.find('\r')); + const auto writeFixture = [](const fs::path& path, const std::string& contents) { + std::ofstream outputStream(path, std::ios::binary | std::ios::trunc); + REQUIRE(outputStream.is_open()); + outputStream << contents; + outputStream.close(); + REQUIRE(outputStream.good()); + }; + writeFixture(exemplarPath, exemplarText); + if(writeGeneratedFile) + { + writeFixture(generatedPath, generatedText); + } - std::string gen = generated.substr(0, generated.find('\n')); - REQUIRE(exem == gen); + const std::string difference = FindINLFileDifference(exemplarPath, generatedPath); + INFO(difference); + if(expectedDifference.empty()) + { + REQUIRE(difference.empty()); + } + else + { + REQUIRE_THAT(difference, Catch::Matchers::Contains(expectedDifference)); } } -} // namespace TEST_CASE("OrientationAnalysis::WriteINLFileFilter: Valid Filter Execution", "[OrientationAnalysis][WriteINLFileFilter]") { @@ -120,7 +251,9 @@ TEST_CASE("OrientationAnalysis::WriteINLFileFilter: Valid Filter Execution", "[O auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); - ::CompareResults(); + const std::string difference = FindINLFileDifference(k_ExemplarFilePath, k_WrittenFilePath); + INFO(difference); + REQUIRE(difference.empty()); UnitTest::CheckArraysInheritTupleDims(dataStructure); } diff --git a/src/Plugins/SimplnxCore/docs/ComputeFeatureCentroidsFilter.md b/src/Plugins/SimplnxCore/docs/ComputeFeatureCentroidsFilter.md index a2ca6d66ce..c8632dde5f 100644 --- a/src/Plugins/SimplnxCore/docs/ComputeFeatureCentroidsFilter.md +++ b/src/Plugins/SimplnxCore/docs/ComputeFeatureCentroidsFilter.md @@ -6,7 +6,7 @@ Generic (Morphological) ## Description -This **Filter** calculates the *centroid* of each **Feature** by determining the average X, Y, and Z position (in physical coordinates) of all the **Cells** belonging to the **Feature**. The per-cell coordinates are accumulated using Kahan compensated summation to limit floating-point round-off on features with large cell counts. An *Is Periodic* option is available: when enabled, a **Feature** that spans the full extent of an axis (touching both opposing faces of the **Image Geometry**) is treated as wrapping around to the opposite face, and its centroid on that axis is computed as a minimum-image (circular) mean — the cell positions are unwrapped across the boundary before averaging and the result is mapped back into the domain. This depends on where the **Feature**'s cells actually lie, so it is correct for features whose mass wraps asymmetrically and never places the centroid outside the sample; a feature that fills the whole domain along an axis falls back to the ordinary average on that axis. When *Is Periodic* is disabled, **Features** that intersect the outer surfaces of the sample will still have *centroids* calculated, but they will be *centroids* of the truncated part of the **Feature** that lies inside the sample. +This **Filter** calculates the *centroid* of each **Feature** by averaging the physical X, Y, and Z positions of all its **Cells**. It uses Kahan compensated summation to limit floating-point round-off. With *Is Periodic* enabled, a feature touching both opposing faces of an axis uses the unique largest empty gap between occupied cell centers as its unwrapping seam. The filter then takes the arithmetic mean of all its unwrapped cell centers, including repeated centers from multiple cells, and maps the result back into the domain. If the largest gap is tied or every cell coordinate on that axis is occupied, the seam is ambiguous and the ordinary mean is retained. Axes with one cell and features that do not touch both faces also retain the ordinary mean. With *Is Periodic* disabled, features intersecting the sample boundary use the ordinary mean of their cells inside the sample. A **Feature** with no **Cells** (an unused Feature Id) keeps a centroid of (0, 0, 0). @@ -16,11 +16,11 @@ A **Feature** with no **Cells** (an unused Feature Id) keeps a centroid of (0, 0 The algorithm iterates over all voxels, accumulating each voxel's XYZ coordinate (computed from its flat index, the geometry origin, and spacing) into a per-feature Kahan sum. After all voxels are processed, each feature's centroid is the accumulated sum divided by the voxel count. -If *Is Periodic* is enabled, a post-processing step checks whether any feature's bounding box spans the full extent of a dimension (indicating it wraps around a periodic boundary) and adjusts the centroid accordingly. +If *Is Periodic* is enabled, a post-processing step checks for features touching both faces of an axis. It records occupied coordinates in packed bitsets, scanning Feature IDs sequentially in batches or axis windows when the available working-memory grant is small. Occupancy determines the seam; a final sequential read uses the original cell counts to compute adjusted components. The filter requests up to 32 MiB of shared working allowance for occupancy and reduces batch or window size to the actual grant. This allowance covers the new occupancy payload only; the existing 256 KiB Feature-ID transfer buffer and feature-sized sums, ranges, output, and candidate metadata are additional memory. A periodic candidate requires at least one 8-byte occupancy word. If that minimum is unavailable, execution reports a resource error before publishing final centroids. ### Performance -This filter is optimized for out-of-core (OOC) data storage. The Feature IDs array is read in fixed-size chunks (64K tuples) via `copyIntoBuffer()` rather than accessing each voxel individually. All accumulation is performed in plain `std::vector` buffers (Kahan sums, compensators, voxel counts, XYZ ranges) to avoid per-element virtual dispatch through the DataStore interface. The final centroids are written to the output array in a single `copyFromBuffer()` call. +The Feature IDs array is read in fixed-size chunks (64K tuples) via `copyIntoBuffer()` rather than accessing each voxel individually. Nonperiodic mode and periodic input with no boundary-spanning feature need only the original algorithm scan. If K occupancy batches or windows are needed, an adjusted result needs K+2 algorithm scans: the original sum, K occupancy scans, and one weighted correction scan. Smaller grants can increase K substantially. Feature-ID validation makes one separate scan. The final centroids are written in one `copyFromBuffer()` call. % Auto generated parameter table will be inserted here diff --git a/src/Plugins/SimplnxCore/docs/ComputeKMeansFilter.md b/src/Plugins/SimplnxCore/docs/ComputeKMeansFilter.md index 956a217c4f..1013ac391d 100644 --- a/src/Plugins/SimplnxCore/docs/ComputeKMeansFilter.md +++ b/src/Plugins/SimplnxCore/docs/ComputeKMeansFilter.md @@ -10,6 +10,8 @@ DREAM3D Review (Clustering) This **Filter** applies the k means algorithm to an **Attribute Array**. K means is a *clustering algorithm* that assigns each point (each tuple) of the **Attribute Array** to a *cluster Id*. A **cluster** is a group of data points that are more similar to each other than to points in other groups. The user must specify the number of clusters (a dimensionless count, *k*) in which to partition the array. +Number of Clusters must be greater than zero. The default is 1. A saved value of zero reports an error. + Each cluster is summarized by its **centroid**, which is the arithmetic mean (the average position) of all the points assigned to that cluster. A k means partitioning is a *Voronoi tessellation*: every point is assigned to the cluster whose centroid is closest to it, so the data space is carved into regions where each region contains all the points nearer to one centroid than to any other. An optimal solution to the k means problem is one in which each point is associated with the cluster that has the closest mean. This partitioning minimizes the **within-cluster variance**, that is, the sum over all clusters of the squared distances from each point to its cluster centroid; minimizing it makes the points within each cluster as tightly grouped as possible. The user may select from several distance metrics: *Euclidean*, *Squared Euclidean*, *Manhattan*, *Cosine*, *Pearson*, and *Squared Pearson*. When **Use Mask Array** is enabled, **Cell Mask Array** must have 1 component and the same number of tuples as **Attribute Array to Cluster**. diff --git a/src/Plugins/SimplnxCore/docs/IdentifySampleFilter.md b/src/Plugins/SimplnxCore/docs/IdentifySampleFilter.md index 7164db69be..40df39c4e2 100644 --- a/src/Plugins/SimplnxCore/docs/IdentifySampleFilter.md +++ b/src/Plugins/SimplnxCore/docs/IdentifySampleFilter.md @@ -46,6 +46,10 @@ When *Process Data Slice-By-Slice* is enabled, the *Slice-By-Slice Plane* parame - **XZ [1]**: Processes the volume slice by slice along the Y axis, scanning each XZ plane independently. - **YZ [2]**: Processes the volume slice by slice along the X axis, scanning each YZ plane independently. +### Out-of-Core YZ Transfers + +For YZ slices, the out-of-core algorithm groups up to eight adjacent slices to reduce repeated reads and writes. Each slice still uses the same classification and hole-filling rules. The group size depends on the memory available for up to seven extra slice buffers. If that memory is unavailable, the algorithm processes one slice at a time. This limit applies to the extra slice buffers, not to all memory used by the filter. + ### Required Input Sources - **Good Voxels Mask** -- a boolean array marking cells as *good* or *bad*, typically produced by [Multi-Threshold Objects](MultiThresholdObjectsFilter.md) applied to EBSD confidence index, image quality, or a similar scalar. diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.cpp index 7aa305b731..bd10f3f4df 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.cpp @@ -1,18 +1,24 @@ #include "ComputeFeatureCentroids.hpp" -#include "simplnx/Common/Constants.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataGroup.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataArrayUtilities.hpp" +#include #include +#include #include #include +#include #include #include #include +#include using namespace nx::core; @@ -20,9 +26,65 @@ namespace { // Each bulk read contains 65,536 Feature IDs. This keeps the staging buffer cache-sized. constexpr usize k_ChunkTuples = 65536; -// A feature with a per-axis unit-vector resultant below this threshold has its mass spread almost uniformly around the domain. -// This distribution makes the circular mean indeterminate. The algorithm keeps the arithmetic mean. -constexpr double k_DegenerateResultant = 1.0e-6; +constexpr uint64 k_PreferredOccupancyBytes = 32ULL * 1024ULL * 1024ULL; +constexpr uint64 k_MinimumOccupancyBytes = sizeof(uint64); +constexpr usize k_NoCandidate = std::numeric_limits::max(); + +// One entry exists only for a feature/axis pair touching both opposing faces. +// The gap fields survive ascending occupancy windows; window fields locate this batch's bits. +// Occupancy selects a seam, while the final Kahan pass retains voxel multiplicities. +struct AxisCandidate +{ + usize FeatureId = 0; + usize Axis = 0; + usize First = 0; + usize Previous = 0; + usize LargestGap = 0; + usize Cut = 0; + usize WindowStart = 0; + usize WindowLength = 0; + usize WordOffset = 0; + bool HasOccupied = false; + bool TiedLargestGap = false; + bool Adjust = false; +}; + +bool WouldMultiplyOverflow(usize lhs, usize rhs) +{ + return rhs != 0 && lhs > std::numeric_limits::max() / rhs; +} + +// Division/remainder avoids overflow from rounding cellCount upward before dividing. +usize WordCount(usize cellCount) +{ + return (cellCount / 64) + static_cast(cellCount % 64 != 0); +} + +// Coordinates arrive in ascending order, including across words and nonempty windows. +// The cut is the occupied coordinate immediately before the largest empty circular gap. +void ObserveOccupiedCoordinate(AxisCandidate& candidate, usize coordinate) +{ + if(!candidate.HasOccupied) + { + candidate.First = coordinate; + candidate.Previous = coordinate; + candidate.HasOccupied = true; + return; + } + + const usize gap = coordinate - candidate.Previous; + if(gap > candidate.LargestGap) + { + candidate.LargestGap = gap; + candidate.Cut = candidate.Previous; + candidate.TiedLargestGap = false; + } + else if(gap == candidate.LargestGap) + { + candidate.TiedLargestGap = true; + } + candidate.Previous = coordinate; +} } // namespace ComputeFeatureCentroids::ComputeFeatureCentroids(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, @@ -48,11 +110,16 @@ Result<> ComputeFeatureCentroids::operator()() auto& centroids = m_DataStructure.getDataAs(m_InputValues->CentroidsArrayPath)->getDataStoreRef(); - auto validateNumFeatResult = ValidateFeatureIdsToFeatureAttributeMatrixIndexing(m_DataStructure, m_InputValues->CentroidsArrayPath, *featureIdsPtr, false, m_MessageHandler); + auto validateNumFeatResult = ValidateFeatureIdsToFeatureAttributeMatrixIndexing(m_DataStructure, m_InputValues->CentroidsArrayPath, *featureIdsPtr, false, m_MessageHandler, &m_ShouldCancel); if(validateNumFeatResult.invalid()) { return validateNumFeatResult; } + Result<> result = std::move(validateNumFeatResult); + if(m_ShouldCancel) + { + return result; + } const auto& imageGeom = m_DataStructure.getDataRefAs(m_InputValues->ImageGeometryPath); const usize totalFeatures = centroids.getNumberOfTuples(); @@ -60,19 +127,22 @@ Result<> ComputeFeatureCentroids::operator()() const usize yPoints = imageGeom.getNumYCells(); const usize zPoints = imageGeom.getNumZCells(); + // The three-component uint64 count vector has the largest feature-sized byte requirement. + const bool featureBytesOverflow = WouldMultiplyOverflow(totalFeatures, 3 * sizeof(uint64)); + if(featureBytesOverflow || WouldMultiplyOverflow(xPoints, yPoints) || WouldMultiplyOverflow(xPoints * yPoints, zPoints)) + { + return MergeResults( + std::move(result), + MakeErrorResult(-5320, fmt::format("ComputeFeatureCentroids cannot index Image Geometry '{}' with dimensions ({}, {}, {}) and {} feature tuples: working-size arithmetic overflows.", + m_InputValues->ImageGeometryPath.toString(), xPoints, yPoints, zPoints, totalFeatures))); + } + // Feature-sized vectors avoid DataStore calls in the voxel accumulation loop. const usize featureElems3 = totalFeatures * 3; const usize featureElems2 = totalFeatures * 2; std::vector kahanSum(featureElems3, 0.0); std::vector kahanComp(featureElems3, 0.0); std::vector voxelCount(featureElems3, 0); - std::vector sumCos; - std::vector sumSin; - if(m_InputValues->IsPeriodic) - { - sumCos.resize(featureElems3, 0.0); - sumSin.resize(featureElems3, 0.0); - } std::vector rangeX(featureElems2, 0); std::vector rangeY(featureElems2, 0); std::vector rangeZ(featureElems2, 0); @@ -86,7 +156,6 @@ Result<> ComputeFeatureCentroids::operator()() const FloatVec3 origin = imageGeom.getOrigin(); const FloatVec3 spacing = imageGeom.getSpacing(); - const std::array domainLength = {static_cast(xPoints) * spacing[0], static_cast(yPoints) * spacing[1], static_cast(zPoints) * spacing[2]}; const usize totalVoxels = xPoints * yPoints * zPoints; const usize xySize = xPoints * yPoints; @@ -96,14 +165,14 @@ Result<> ComputeFeatureCentroids::operator()() { if(m_ShouldCancel) { - return {}; + return result; } const usize chunkCount = std::min(k_ChunkTuples, totalVoxels - offset); - Result<> ioResult = featureIdsStoreRef.copyIntoBuffer(offset, nonstd::span(featureIdBuf.get(), chunkCount)); - if(ioResult.invalid()) + result = MergeResults(std::move(result), featureIdsStoreRef.copyIntoBuffer(offset, nonstd::span(featureIdBuf.get(), chunkCount))); + if(result.invalid()) { - return ioResult; + return result; } for(usize idx = 0; idx < chunkCount; idx++) @@ -122,11 +191,11 @@ Result<> ComputeFeatureCentroids::operator()() const usize fid = static_cast(featureId); rangeX[fid * 2] = std::min(k, rangeX[fid * 2]); - rangeX[fid * 2 + 1] = std::max(k, rangeX[fid * 2 + 1]); + rangeX[(fid * 2) + 1] = std::max(k, rangeX[(fid * 2) + 1]); rangeY[fid * 2] = std::min(j, rangeY[fid * 2]); - rangeY[fid * 2 + 1] = std::max(j, rangeY[fid * 2 + 1]); + rangeY[(fid * 2) + 1] = std::max(j, rangeY[(fid * 2) + 1]); rangeZ[fid * 2] = std::min(i, rangeZ[fid * 2]); - rangeZ[fid * 2 + 1] = std::max(i, rangeZ[fid * 2 + 1]); + rangeZ[(fid * 2) + 1] = std::max(i, rangeZ[(fid * 2) + 1]); const double vx = static_cast(origin[0]) + (static_cast(k) + 0.5) * static_cast(spacing[0]); const double vy = static_cast(origin[1]) + (static_cast(j) + 0.5) * static_cast(spacing[1]); @@ -141,12 +210,6 @@ Result<> ComputeFeatureCentroids::operator()() kahanComp[fi] = (temp - kahanSum[fi]) - componentValue; kahanSum[fi] = temp; voxelCount[fi]++; - if(m_InputValues->IsPeriodic) - { - const double phase = Constants::k_2Pi * (voxelCoords[c] - static_cast(origin[c])) / domainLength[c]; - sumCos[fi] += std::cos(phase); - sumSin[fi] += std::sin(phase); - } } } } @@ -167,52 +230,246 @@ Result<> ComputeFeatureCentroids::operator()() if(m_InputValues->IsPeriodic) { m_MessageHandler.sendMessage({IFilter::Message::Type::Info, "Checking for periodic data."}); - // When a feature spans the full extent on an axis, its arithmetic centroid is in the empty middle of the wrapped feature. - // Replace that component with the circular mean from the per-feature unit-vector sums. - // If the resultant is near zero, the feature mass is almost uniform and the algorithm keeps the arithmetic mean. - const std::array*, 3> rangeStores = {&rangeX, &rangeY, &rangeZ}; + // Reuse the range minima as a constant-time feature/axis-to-candidate lookup. + std::array*, 3> rangeStores = {&rangeX, &rangeY, &rangeZ}; const std::array dims = {xPoints, yPoints, zPoints}; - bool anyAdjusted = false; + std::vector candidates; + usize usefulWords = 0; + constexpr usize preferredWords = static_cast(k_PreferredOccupancyBytes / sizeof(uint64)); for(usize featureId = 0; featureId < totalFeatures; featureId++) { for(usize axis = 0; axis < 3; axis++) { - const usize axisIdx = featureId * 3 + axis; - if(voxelCount[axisIdx] == 0) + const usize axisIdx = (featureId * 3) + axis; + auto& rangeStore = *rangeStores[axis]; + const bool spansExtent = voxelCount[axisIdx] != 0 && dims[axis] > 1 && rangeStore[featureId * 2] == 0 && rangeStore[(featureId * 2) + 1] == dims[axis] - 1; + if(spansExtent) { - continue; + const usize candidateWords = WordCount(dims[axis]); + usefulWords += std::min(candidateWords, preferredWords - usefulWords); + rangeStore[featureId * 2] = candidates.size(); + AxisCandidate candidate; + candidate.FeatureId = featureId; + candidate.Axis = axis; + candidates.push_back(candidate); } - const auto& rangeStore = *rangeStores[axis]; - const bool spansExtent = rangeStore[featureId * 2] == 0 && rangeStore[featureId * 2 + 1] == dims[axis] - 1; - if(!spansExtent) + else { - continue; + rangeStore[featureId * 2] = k_NoCandidate; } - const double resultant = std::sqrt(sumCos[axisIdx] * sumCos[axisIdx] + sumSin[axisIdx] * sumSin[axisIdx]) / static_cast(voxelCount[axisIdx]); - if(resultant < k_DegenerateResultant) + } + } + + if(!candidates.empty()) + { + { + auto reservation = CacheMemoryBudgetManager::instance().reserveWorkingMemory(static_cast(usefulWords) * sizeof(uint64)); + if(reservation.sizeBytes() < k_MinimumOccupancyBytes) + { + return MergeResults(std::move(result), MakeErrorResult(-5321, fmt::format("ComputeFeatureCentroids cannot scan periodic Feature IDs at '{}': shared working allowance granted {} bytes; at " + "least {} bytes are required. Retry after other working operations finish.", + m_InputValues->FeatureIdsArrayPath.toString(), reservation.sizeBytes(), k_MinimumOccupancyBytes))); + } + + const usize availableWords = static_cast(reservation.sizeBytes() / sizeof(uint64)); + auto occupancy = std::make_unique>(availableWords); + usize nextCandidate = 0; + usize nextCoordinate = 0; + usize occupancyScans = 0; + while(nextCandidate < candidates.size()) + { + if(m_ShouldCancel) + { + return result; + } + + const usize firstCandidate = nextCandidate; + usize lastCandidateExclusive = firstCandidate; + usize usedWords = 0; + // Full candidates share a batch. An oversized candidate resumes in ascending windows. + while(nextCandidate < candidates.size() && usedWords < availableWords) + { + auto& candidate = candidates[nextCandidate]; + const usize remaining = dims[candidate.Axis] - nextCoordinate; + const usize windowLength = std::min(remaining, (availableWords - usedWords) * 64); + candidate.WindowStart = nextCoordinate; + candidate.WindowLength = windowLength; + candidate.WordOffset = usedWords; + usedWords += WordCount(windowLength); + lastCandidateExclusive = nextCandidate + 1; + if(windowLength == remaining) + { + nextCandidate++; + nextCoordinate = 0; + } + else + { + nextCoordinate += windowLength; + break; + } + } + + std::fill_n(occupancy->data(), usedWords, uint64{0}); + for(usize offset = 0; offset < totalVoxels;) + { + if(m_ShouldCancel) + { + return result; + } + const usize chunkCount = std::min(k_ChunkTuples, totalVoxels - offset); + result = MergeResults(std::move(result), featureIdsStoreRef.copyIntoBuffer(offset, nonstd::span(featureIdBuf.get(), chunkCount))); + if(result.invalid()) + { + return result; + } + for(usize idx = 0; idx < chunkCount; idx++) + { + const int32 featureId = featureIdBuf[idx]; + if(featureId < 0) + { + continue; + } + const usize flatIdx = offset + idx; + const std::array coordinates = {flatIdx % xPoints, (flatIdx / xPoints) % yPoints, flatIdx / xySize}; + const usize fid = static_cast(featureId); + for(usize axis = 0; axis < 3; axis++) + { + const usize candidateIndex = static_cast((*rangeStores[axis])[fid * 2]); + if(candidateIndex < firstCandidate || candidateIndex >= lastCandidateExclusive) + { + continue; + } + const auto& candidate = candidates[candidateIndex]; + const usize coordinate = coordinates[axis]; + if(coordinate >= candidate.WindowStart && coordinate - candidate.WindowStart < candidate.WindowLength) + { + const usize local = coordinate - candidate.WindowStart; + (*occupancy)[candidate.WordOffset + (local / 64)] |= uint64{1} << (local % 64); + } + } + } + offset += chunkCount; + } + + for(usize candidateIndex = firstCandidate; candidateIndex < lastCandidateExclusive; candidateIndex++) + { + auto& candidate = candidates[candidateIndex]; + const usize wordCount = WordCount(candidate.WindowLength); + for(usize wordIndex = 0; wordIndex < wordCount; wordIndex++) + { + if((wordIndex % 1024 == 0) && m_ShouldCancel) + { + return result; + } + uint64 word = (*occupancy)[candidate.WordOffset + wordIndex]; + while(word != 0) + { + const usize bit = static_cast(std::countr_zero(word)); + ObserveOccupiedCoordinate(candidate, candidate.WindowStart + (wordIndex * 64) + bit); + word &= word - 1; + } + } + } + occupancyScans++; + if(std::has_single_bit(occupancyScans)) + { + m_MessageHandler.sendMessage({IFilter::Message::Type::Info, fmt::format("ComputeFeatureCentroids completed {} periodic occupancy scan(s).", occupancyScans)}); + } + } + } + + bool anyAdjusted = false; + for(auto& candidate : candidates) + { + if(!candidate.HasOccupied) { continue; } - double phase = std::atan2(sumSin[axisIdx], sumCos[axisIdx]); - if(phase < 0.0) + const usize closingGap = dims[candidate.Axis] - candidate.Previous + candidate.First; + if(closingGap > candidate.LargestGap) { - phase += Constants::k_2Pi; + candidate.LargestGap = closingGap; + candidate.TiedLargestGap = false; + } + else if(closingGap == candidate.LargestGap) + { + candidate.TiedLargestGap = true; + } + candidate.Adjust = candidate.LargestGap > 1 && !candidate.TiedLargestGap; + if(candidate.Adjust) + { + const usize fi = (candidate.FeatureId * 3) + candidate.Axis; + kahanSum[fi] = 0.0; + kahanComp[fi] = 0.0; + anyAdjusted = true; } - centroidsBuf[axisIdx] = static_cast(static_cast(origin[axis]) + (phase / Constants::k_2PiD) * domainLength[axis]); - anyAdjusted = true; } - } - if(anyAdjusted) - { - m_MessageHandler.sendMessage({IFilter::Message::Type::Info, "ComputeFeatureCentroids adjusted centroids of features that wrap the periodic boundary."}); + + if(anyAdjusted) + { + for(usize offset = 0; offset < totalVoxels;) + { + if(m_ShouldCancel) + { + return result; + } + const usize chunkCount = std::min(k_ChunkTuples, totalVoxels - offset); + result = MergeResults(std::move(result), featureIdsStoreRef.copyIntoBuffer(offset, nonstd::span(featureIdBuf.get(), chunkCount))); + if(result.invalid()) + { + return result; + } + for(usize idx = 0; idx < chunkCount; idx++) + { + const int32 featureId = featureIdBuf[idx]; + if(featureId < 0) + { + continue; + } + const usize flatIdx = offset + idx; + const std::array coordinates = {flatIdx % xPoints, (flatIdx / xPoints) % yPoints, flatIdx / xySize}; + const usize fid = static_cast(featureId); + for(usize axis = 0; axis < 3; axis++) + { + const usize candidateIndex = static_cast((*rangeStores[axis])[fid * 2]); + if(candidateIndex == k_NoCandidate || !candidates[candidateIndex].Adjust) + { + continue; + } + const auto& candidate = candidates[candidateIndex]; + const double unwrapped = static_cast(coordinates[axis]) + 0.5 + (coordinates[axis] <= candidate.Cut ? static_cast(dims[axis]) : 0.0); + const usize fi = (fid * 3) + axis; + const double componentValue = unwrapped - kahanComp[fi]; + const double temp = kahanSum[fi] + componentValue; + kahanComp[fi] = (temp - kahanSum[fi]) - componentValue; + kahanSum[fi] = temp; + } + } + offset += chunkCount; + } + + for(const auto& candidate : candidates) + { + if(!candidate.Adjust) + { + continue; + } + const usize fi = (candidate.FeatureId * 3) + candidate.Axis; + double wrappedIndexCenter = std::fmod(kahanSum[fi] / static_cast(voxelCount[fi]), static_cast(dims[candidate.Axis])); + if(wrappedIndexCenter < 0.0) + { + wrappedIndexCenter += static_cast(dims[candidate.Axis]); + } + centroidsBuf[fi] = static_cast(static_cast(origin[candidate.Axis]) + (static_cast(spacing[candidate.Axis]) * wrappedIndexCenter)); + } + m_MessageHandler.sendMessage({IFilter::Message::Type::Info, "ComputeFeatureCentroids adjusted centroids of features that wrap the periodic boundary."}); + } } } - Result<> writeResult = centroids.copyFromBuffer(0, nonstd::span(centroidsBuf.data(), featureElems3)); - if(writeResult.invalid()) + if(m_ShouldCancel) { - return writeResult; + return result; } - - return {}; + return MergeResults(std::move(result), centroids.copyFromBuffer(0, nonstd::span(centroidsBuf.data(), featureElems3))); } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.hpp index 275fd04fe2..a55c72bfb0 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.hpp @@ -32,11 +32,9 @@ struct SIMPLNXCORE_EXPORT ComputeFeatureCentroidsInputValues * @brief Computes ImageGeom feature centroids with Kahan summation. * * Feature IDs use 65,536-tuple bulk reads. Kahan sums, counts, and periodic ranges use - * feature-sized vectors. This avoids per-voxel DataStore access without cell-sized resident memory. - * The centroid array uses one final bulk write. - * - * Current bulk-I/O Result values are not inspected. A storage failure can leave output state while - * the method returns success. + * feature-sized vectors. The periodic path uses a bounded occupied-coordinate bitset and + * an additional weighted Kahan pass for adjusted axes. It does not retain per-voxel + * coordinates in memory. The centroid array uses one final bulk write. */ class SIMPLNXCORE_EXPORT ComputeFeatureCentroids { @@ -63,10 +61,10 @@ class SIMPLNXCORE_EXPORT ComputeFeatureCentroids /** * @brief Computes feature centroids. - * @return Success, or a Feature ID indexing-validation error. + * @return Success or a validation, working-memory, or store I/O error. * - * When a chunk checkpoint observes cancellation, the method returns success before the centroid - * bulk write. The periodic adjustment does not check cancellation after that write starts. + * When a checkpoint observes cancellation, the method returns before the final centroid + * bulk write. A failed final write follows the store's existing partial-write contract. */ Result<> operator()(); diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.cpp index addc53e529..db4ecc5886 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.cpp @@ -2,11 +2,14 @@ #include "ComputeKMeans.hpp" +#include "simplnx/Common/Result.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/Utilities/ClusteringUtilities.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" #include "simplnx/Utilities/MaskCompareUtilities.hpp" +#include + #include using namespace nx::core; @@ -304,6 +307,11 @@ const std::atomic_bool& ComputeKMeansDirect::getCancel() // ----------------------------------------------------------------------------- Result<> ComputeKMeansDirect::operator()() { + if(m_InputValues->InitClusters == 0) + { + return MakeErrorResult(-54061, fmt::format("Number of Clusters ({}) for array '{}' must be greater than zero.", m_InputValues->InitClusters, m_InputValues->ClusteringArrayPath.toString())); + } + auto* clusteringArray = m_DataStructure.getDataAs(m_InputValues->ClusteringArrayPath); std::unique_ptr maskCompare; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.hpp index 2982cd4ab5..f1b43e2bf1 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.hpp @@ -38,13 +38,14 @@ class SIMPLNXCORE_EXPORT ComputeKMeansDirect /** * @brief Executes K-Means with direct element access. - * @return Success, or an invalid-mask or empty-mask error. + * @return Success, or an invalid mask, empty mask, or zero cluster count error. * @pre The input has at least one tuple. Its component count is positive and fits in Int32. - * @pre The cluster count is positive, fits in Int32, and output shapes are compatible. + * @pre A positive cluster count fits in Int32, and output shapes are compatible. * - * This path does not validate these shape preconditions. It does not check - * cancellation during centroid selection. Later cancellation returns success - * and can leave partial assignments or centroids. + * This path rejects zero clusters. Other limits remain preconditions. + * The selection loop does not check + * cancellation. + * Later cancellation returns success. Assignments or centroids can be partial. */ Result<> operator()(); diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.cpp index aafa5e9050..b93b94eaf4 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.cpp @@ -472,6 +472,11 @@ const std::atomic_bool& ComputeKMeansScanline::getCancel() Result<> ComputeKMeansScanline::operator()() { + if(m_InputValues->InitClusters == 0) + { + return MakeErrorResult(-54061, fmt::format("Number of Clusters ({}) for array '{}' must be greater than zero.", m_InputValues->InitClusters, m_InputValues->ClusteringArrayPath.toString())); + } + const auto& clusteringArray = m_DataStructure.getDataRefAs(m_InputValues->ClusteringArrayPath); const IDataArray* maskArray = m_InputValues->UseMask ? m_DataStructure.getDataAs(m_InputValues->MaskArrayPath) : nullptr; auto& meansArray = m_DataStructure.getDataRefAs(m_InputValues->MeansArrayPath); diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.hpp index ca479f56fe..19bdbef6ea 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.hpp @@ -39,7 +39,7 @@ class SIMPLNXCORE_EXPORT ComputeKMeansScanline /** * @brief Executes K-Means with bounded bulk transfers. - * @return Success, or a mask, shape, overflow, or bulk-transfer error. + * @return Success, or a zero cluster count, mask, shape, overflow, or bulk-transfer error. * * Cancellation returns success. Completed assignment pages and centroids from * earlier phases remain in the output arrays. diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.cpp index d2c33780a4..948790a4b9 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.cpp @@ -13,22 +13,170 @@ #include "IdentifySample.hpp" #include "IdentifySampleCommon.hpp" +#include "simplnx/Common/Array.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/DataStructure/IO/Generic/ITemporaryRecordStore.hpp" #include "simplnx/Utilities/BoundedRecordPageCache.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include "simplnx/Utilities/ExternalEquivalence.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" #include "simplnx/Utilities/InMemoryTemporaryRecordStore.hpp" #include "simplnx/Utilities/ThrottledMessageHandler.hpp" +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include +#include +#include using namespace nx::core; +#if SIMPLNX_BUILD_TESTS +namespace +{ +thread_local IdentifySampleSliceObserverForTesting IdentifySampleSliceObserver; +thread_local IdentifySampleExtraAllocationControlForTesting IdentifySampleExtraAllocationControl; +thread_local IdentifySampleDiagnosticControlForTesting IdentifySampleDiagnosticControl; + +void ObserveIdentifySampleSlice(const IDataArray* mask, IdentifySampleSliceEventForTesting event, usize values, usize elementBytes, bool success) noexcept +{ + if(IdentifySampleSliceObserver.callback != nullptr) + { + IdentifySampleSliceObserver.callback(IdentifySampleSliceObserver.context, mask, event, values, elementBytes, success); + } +} + +/** + * @class IdentifySampleCarrierObservation + * @brief Records the lifetime of one original slice carrier. + * @note Declare this owner before the carrier so its release event follows destruction of the value buffer. + */ +class IdentifySampleCarrierObservation +{ +public: + /** + * @brief Creates an inactive owner before its carrier allocates. + * @param mask Borrows the selected array identity. + * @param isXy True for the XY carrier, false for the original plane carrier. + * @param elementBytes Bytes in each allocated value. + */ + IdentifySampleCarrierObservation(const IDataArray* mask, bool isXy, usize elementBytes) noexcept + : m_Mask(mask) + , m_IsXy(isXy) + , m_ElementBytes(elementBytes) + { + } + + /** @brief Reports release after the corresponding value buffer has been destroyed. */ + ~IdentifySampleCarrierObservation() noexcept + { + if(m_Allocated) + { + ObserveIdentifySampleSlice(m_Mask, m_IsXy ? IdentifySampleSliceEventForTesting::XyReleased : IdentifySampleSliceEventForTesting::PlaneReleased, m_Values, m_ElementBytes, true); + } + } + + /** + * @brief Reports successful allocation from the actual allocation site. + * @param values Number of values that the carrier allocated. + */ + void acquired(usize values) noexcept + { + m_Values = values; + m_Allocated = true; + ObserveIdentifySampleSlice(m_Mask, m_IsXy ? IdentifySampleSliceEventForTesting::XyAllocated : IdentifySampleSliceEventForTesting::PlaneAllocated, values, m_ElementBytes, true); + } + + IdentifySampleCarrierObservation(const IdentifySampleCarrierObservation&) = delete; + IdentifySampleCarrierObservation& operator=(const IdentifySampleCarrierObservation&) = delete; + IdentifySampleCarrierObservation(IdentifySampleCarrierObservation&&) = delete; + IdentifySampleCarrierObservation& operator=(IdentifySampleCarrierObservation&&) = delete; + +private: + const IDataArray* m_Mask; + bool m_IsXy; + usize m_ElementBytes; + usize m_Values = 0; + bool m_Allocated = false; +}; + +/** + * @brief Observes the original read exactly once and reports returned failures or exception exit. + * @tparam T Mask element type. + * @param mask Borrows the selected array identity. + * @param store Original mask store. + * @param start First flat value index. + * @param output Receives mask values. + * @return The original read Result. + */ +template +Result<> ObserveIdentifySampleRead(const IDataArray* mask, AbstractDataStore& store, usize start, nonstd::span output) +{ + bool success = false; + ObserveIdentifySampleSlice(mask, IdentifySampleSliceEventForTesting::ReadBegin, output.size(), sizeof(T), false); + const auto complete = MakeScopeGuard([&]() noexcept { ObserveIdentifySampleSlice(mask, IdentifySampleSliceEventForTesting::ReadEnd, output.size(), sizeof(T), success); }); + auto result = store.copyIntoBuffer(start, output); + success = result.valid(); + return result; +} + +/** + * @brief Observes the original write exactly once and reports returned failures or exception exit. + * @tparam T Mask element type. + * @param mask Borrows the selected array identity. + * @param store Original mask store. + * @param start First flat value index. + * @param input Supplies mask values. + * @return The original write Result. + */ +template +Result<> ObserveIdentifySampleWrite(const IDataArray* mask, AbstractDataStore& store, usize start, nonstd::span input) +{ + bool success = false; + ObserveIdentifySampleSlice(mask, IdentifySampleSliceEventForTesting::WriteBegin, input.size(), sizeof(T), false); + const auto complete = MakeScopeGuard([&]() noexcept { ObserveIdentifySampleSlice(mask, IdentifySampleSliceEventForTesting::WriteEnd, input.size(), sizeof(T), success); }); + auto result = store.copyFromBuffer(start, input); + success = result.valid(); + return result; +} +} // namespace + +IdentifySampleSliceObserverForTesting nx::core::SetIdentifySampleSliceObserverForTesting(IdentifySampleSliceObserverForTesting observer) noexcept +{ + return std::exchange(IdentifySampleSliceObserver, observer); +} + +IdentifySampleExtraAllocationControlForTesting nx::core::SetIdentifySampleExtraAllocationControlForTesting(IdentifySampleExtraAllocationControlForTesting control) noexcept +{ + return std::exchange(IdentifySampleExtraAllocationControl, control); +} + +IdentifySampleDiagnosticControlForTesting nx::core::SetIdentifySampleDiagnosticControlForTesting(IdentifySampleDiagnosticControlForTesting control) noexcept +{ + return std::exchange(IdentifySampleDiagnosticControl, control); +} + +#define NX_IDENTIFY_SAMPLE_SLICE_READ(mask, store, start, output) ObserveIdentifySampleRead(mask, store, start, output) +#define NX_IDENTIFY_SAMPLE_SLICE_WRITE(mask, store, start, input) ObserveIdentifySampleWrite(mask, store, start, input) +#else +#define NX_IDENTIFY_SAMPLE_SLICE_READ(mask, store, start, output) (store).copyIntoBuffer(start, output) +#define NX_IDENTIFY_SAMPLE_SLICE_WRITE(mask, store, start, input) (store).copyFromBuffer(start, input) +#endif + namespace { /** @@ -726,6 +874,202 @@ Result<> identifyPlane(T* data, int64 dim1, int64 dim2, bool fillHoles, bool all shouldCancel); } +/** + * @brief Multiplies slice counts without unsigned overflow. + * @param left First count. + * @param right Second count. + * @param maskName Identifies the selected mask. + * @return Product, or an error with the operands and supported bound. + */ +Result CheckedSliceProduct(usize left, usize right, const std::string& maskName) +{ + if(right != 0 && left > std::numeric_limits::max() / right) + { + return MakeErrorResult(-45463, fmt::format("Identify Sample mask '{}': product {} * {} exceeds the supported value {}.", maskName, left, right, std::numeric_limits::max())); + } + return {left * right}; +} + +/** + * @struct IdentifySampleSliceLayout + * @brief Holds checked unsigned transfer sizes and signed CCL dimensions. + */ +struct IdentifySampleSliceLayout +{ + usize X = 0; + usize Y = 0; + usize Z = 0; + usize XyValues = 0; + usize PlaneValues = 0; + usize PlaneBytes = 0; + usize FixedCount = 0; + int64 PlaneDim1 = 0; + int64 PlaneDim2 = 0; +}; + +/** + * @brief Checks slice offsets, value buffers, and the unchanged CCL label rows. + * @param dimensions Image dimensions in X, Y, Z order. + * @param plane Selected plane orientation. + * @param elementBytes Bytes in one mask value. + * @param maskName Identifies the selected mask. + * @return Safe layout, or an error before allocation and transfer. + */ +Result MakeIdentifySampleSliceLayout(const SizeVec3& dimensions, IdentifySampleSliceBySliceFunctor::Plane plane, usize elementBytes, const std::string& maskName) +{ + for(usize axis = 0; axis < 3; ++axis) + { + if(dimensions[axis] > static_cast(std::numeric_limits::max())) + { + return MakeErrorResult( + -45464, fmt::format("Identify Sample mask '{}': axis {} size {} exceeds the signed CCL bound {}.", maskName, axis, dimensions[axis], std::numeric_limits::max())); + } + } + IdentifySampleSliceLayout layout; + layout.X = dimensions[0]; + layout.Y = dimensions[1]; + layout.Z = dimensions[2]; + if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY) + { + layout.PlaneDim1 = static_cast(layout.X); + layout.PlaneDim2 = static_cast(layout.Y); + layout.FixedCount = layout.Z; + } + else if(plane == IdentifySampleSliceBySliceFunctor::Plane::XZ) + { + layout.PlaneDim1 = static_cast(layout.X); + layout.PlaneDim2 = static_cast(layout.Z); + layout.FixedCount = layout.Y; + } + else + { + layout.PlaneDim1 = static_cast(layout.Y); + layout.PlaneDim2 = static_cast(layout.Z); + layout.FixedCount = layout.X; + } + auto xyResult = CheckedSliceProduct(layout.X, layout.Y, maskName); + if(xyResult.invalid()) + { + return ConvertInvalidResult(std::move(xyResult)); + } + layout.XyValues = xyResult.value(); + auto volumeResult = CheckedSliceProduct(layout.XyValues, layout.Z, maskName); + if(volumeResult.invalid()) + { + return ConvertInvalidResult(std::move(volumeResult)); + } + auto volumeBytesResult = CheckedSliceProduct(volumeResult.value(), elementBytes, maskName); + if(volumeBytesResult.invalid()) + { + return ConvertInvalidResult(std::move(volumeBytesResult)); + } + auto planeResult = CheckedSliceProduct(static_cast(layout.PlaneDim1), static_cast(layout.PlaneDim2), maskName); + if(planeResult.invalid()) + { + return ConvertInvalidResult(std::move(planeResult)); + } + layout.PlaneValues = planeResult.value(); + if(layout.PlaneValues > static_cast(std::numeric_limits::max()) || layout.PlaneDim1 > std::numeric_limits::max() / 2) + { + return MakeErrorResult(-45464, fmt::format("Identify Sample mask '{}': plane {} * {} must fit {} values, and its first dimension must not exceed {} for two label rows.", + maskName, layout.PlaneDim1, layout.PlaneDim2, std::numeric_limits::max(), std::numeric_limits::max() / 2)); + } + auto planeBytesResult = CheckedSliceProduct(layout.PlaneValues, elementBytes, maskName); + if(planeBytesResult.invalid()) + { + return ConvertInvalidResult(std::move(planeBytesResult)); + } + layout.PlaneBytes = planeBytesResult.value(); + auto xyBytesResult = CheckedSliceProduct(layout.XyValues, elementBytes, maskName); + if(xyBytesResult.invalid()) + { + return ConvertInvalidResult(std::move(xyBytesResult)); + } + const usize allocationBound = static_cast(std::numeric_limits::max()); + if(layout.PlaneBytes > allocationBound || (plane != IdentifySampleSliceBySliceFunctor::Plane::XY && plane != IdentifySampleSliceBySliceFunctor::Plane::XZ && xyBytesResult.value() > allocationBound)) + { + return MakeErrorResult(-45465, fmt::format("Identify Sample mask '{}': plane buffer {} bytes or XY buffer {} bytes exceeds the allocation bound {} bytes.", maskName, + layout.PlaneBytes, xyBytesResult.value(), allocationBound)); + } + auto labelCountResult = CheckedSliceProduct(usize{2}, static_cast(layout.PlaneDim1), maskName); + if(labelCountResult.invalid()) + { + return ConvertInvalidResult(std::move(labelCountResult)); + } + auto labelBytesResult = CheckedSliceProduct(labelCountResult.value(), sizeof(int64), maskName); + if(labelBytesResult.invalid()) + { + return ConvertInvalidResult(std::move(labelBytesResult)); + } + const usize labelBound = std::vector().max_size(); + if(labelCountResult.value() > labelBound) + { + return MakeErrorResult(-45465, fmt::format("Identify Sample mask '{}': two label rows of {} values ({} bytes) exceed the allocation bound {} values.", maskName, + labelCountResult.value(), labelBytesResult.value(), labelBound)); + } + return {layout}; +} + +/** + * @enum DeferredSlicePhase + * @brief Identifies an operation whose failure follows completed planes. + */ +enum class DeferredSlicePhase +{ + ReportPlane, ///< Formats or sends the current Slice message. + ClassifyPlane, ///< Runs the unchanged per-plane CCL. + ReadPrefix, ///< Reads an XY slice for completed-plane publication. + WritePrefix ///< Writes an XY slice with completed planes. +}; + +/** + * @struct DeferredSliceFailureLocation + * @brief Stores failure context without formatting while an exception is captured. + */ +struct DeferredSliceFailureLocation +{ + DeferredSlicePhase Phase = DeferredSlicePhase::ClassifyPlane; + usize PlaneIndex = 0; + usize FirstCompletedPlane = 0; + usize CompletedPlaneCount = 0; + std::optional CommitZ; +}; + +/** + * @brief Converts an exception only when a plane step and its prefix publication both fail. + * @param failure Captured non-null exception. + * @param maskName Identifies the selected mask. + * @param location Records the failed operation and its actual indices. + * @return Owned diagnostic with the existing filter exception classification. + * @note Diagnostic allocation can throw; the caller retains both original failures. + */ +Result<> MakeDeferredSliceExceptionResult(const std::exception_ptr& failure, std::string_view maskName, const DeferredSliceFailureLocation& location) +{ + std::string context; + if(location.Phase == DeferredSlicePhase::ReportPlane || location.Phase == DeferredSlicePhase::ClassifyPlane) + { + context = fmt::format("{} {}", location.Phase == DeferredSlicePhase::ReportPlane ? "reporting plane" : "classifying plane", location.PlaneIndex); + } + else + { + const char* phasePtr = location.Phase == DeferredSlicePhase::ReadPrefix ? "reading completed prefix" : "writing completed prefix"; + context = fmt::format("{} starting at plane {} with {} planes at Z {}", phasePtr, location.FirstCompletedPlane, location.CompletedPlaneCount, location.CommitZ.value()); + } + try + { + std::rethrow_exception(failure); + } catch(const std::bad_alloc&) + { + return MakeErrorResult(-272, fmt::format("Identify Sample mask '{}': allocation failure while {}.", maskName, context)); + } catch(const std::exception& exception) + { + return MakeErrorResult(-2, fmt::format("Identify Sample mask '{}': exception while {}: {}", maskName, context, exception.what())); + } catch(...) + { + return MakeErrorResult(-2, fmt::format("Identify Sample mask '{}': nonstandard exception while {}.", maskName, context)); + } +} + /** * @struct IdentifySampleSliceCCLFunctor * @brief Dispatches bounded slice-by-slice CCL by mask value type. @@ -733,7 +1077,7 @@ Result<> identifyPlane(T* data, int64 dim1, int64 dim2, bool fillHoles, bool all struct IdentifySampleSliceCCLFunctor { /** - * @brief Processes selected planes with one plane buffer at a time. + * @brief Classifies planes serially and groups YZ transfers within admitted extra memory. * @tparam T Mask value type. * @param imageGeom Image geometry that supplies dimensions. * @param maskArray Mask array modified in place. @@ -741,143 +1085,364 @@ struct IdentifySampleSliceCCLFunctor * @param plane Selected plane orientation. * @param messageHandler Receives slice progress messages. * @param shouldCancel Cancellation flag. - * @return Bulk-I/O, CCL, or cancellation result. + * @return Bulk-I/O, CCL, or cancellation result; combined failures retain primary-first order. + * @note A lone exception propagates unchanged. Diagnostic assembly failure preserves the original primary and can lose secondary detail. * - * XY planes transfer contiguously. XZ planes transfer contiguous rows. YZ - * planes bulk-read Z slices before extracting and updating one column. + * XY transfers whole planes; XZ transfers rows. YZ groups at most eight planes + * and publishes only completed planes through fresh XY reads. The reservation + * covers extra YZ values, not the original carriers, CCL records, or backend memory. + * For an out-of-core mask the first YZ batch holds one plane and no extra-plane + * reservation, so the first mask write never competes with the extra planes. */ template Result<> operator()(const ImageGeom* imageGeom, IDataArray* maskArray, bool fillHoles, IdentifySampleSliceBySliceFunctor::Plane plane, const IFilter::MessageHandler& messageHandler, const std::atomic_bool& shouldCancel) const { - auto& store = maskArray->template getIDataStoreRefAs>(); - const bool allowInMemoryFallback = store.getStoreType() != IDataStore::StoreType::OutOfCore; - const SizeVec3 dimensions = imageGeom->getDimensions(); - const int64 dimX = static_cast(dimensions[0]); - const int64 dimY = static_cast(dimensions[1]); - const int64 dimZ = static_cast(dimensions[2]); - const usize zSliceSize = static_cast(dimX * dimY); - - int64 planeDim1 = 0; - int64 planeDim2 = 0; - int64 fixedDim = 0; - if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY) + if(shouldCancel) { - planeDim1 = dimX; - planeDim2 = dimY; - fixedDim = dimZ; + return {}; } - else if(plane == IdentifySampleSliceBySliceFunctor::Plane::XZ) + auto& store = maskArray->template getIDataStoreRefAs>(); + const bool allowInMemoryFallback = store.getStoreType() != IDataStore::StoreType::OutOfCore; + const auto maskName = maskArray->getName(); + auto layoutResult = MakeIdentifySampleSliceLayout(imageGeom->getDimensions(), plane, sizeof(T), maskName); + if(layoutResult.invalid()) { - planeDim1 = dimX; - planeDim2 = dimZ; - fixedDim = dimY; + return ConvertResult(std::move(layoutResult)); } - else + const auto& layout = layoutResult.value(); + if(layout.FixedCount == 0) { - planeDim1 = dimY; - planeDim2 = dimZ; - fixedDim = dimX; + return {}; } - const usize planeSize = static_cast(planeDim1 * planeDim2); - auto planeBuffer = std::make_unique(planeSize); - auto zBuffer = std::make_unique(zSliceSize); - - ThrottledMessageHandler progressThrottle(messageHandler); - progressThrottle.reset(static_cast(fixedDim), "Identifying Sample Slices"); - for(int64 fixed = 0; fixed < fixedDim; fixed++) + constexpr usize maxYzBatchPlanes = 8; + const bool isYz = plane != IdentifySampleSliceBySliceFunctor::Plane::XY && plane != IdentifySampleSliceBySliceFunctor::Plane::XZ; + if(isYz && layout.PlaneBytes != 0) { - if(shouldCancel) + const usize maximumExtras = std::min(maxYzBatchPlanes - 1, layout.FixedCount - 1); + auto extraValuesResult = CheckedSliceProduct(maximumExtras, layout.PlaneValues, maskName); + if(extraValuesResult.invalid()) { - return {}; + return ConvertResult(std::move(extraValuesResult)); } - if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY) + auto extraBytesResult = CheckedSliceProduct(extraValuesResult.value(), sizeof(T), maskName); + if(extraBytesResult.invalid()) { - auto readResult = store.copyIntoBuffer(static_cast(fixed) * planeSize, nonstd::span(planeBuffer.get(), planeSize)); - if(readResult.invalid()) + return ConvertResult(std::move(extraBytesResult)); + } + if constexpr(std::numeric_limits::digits > std::numeric_limits::digits) + { + if(extraBytesResult.value() > std::numeric_limits::max()) { - return readResult; + return MakeErrorResult(-45465, fmt::format("Identify Sample mask '{}': {} extra planes * {} bytes exceeds the reservation bound {} bytes.", maskName, maximumExtras, layout.PlaneBytes, + std::numeric_limits::max())); } } - else if(plane == IdentifySampleSliceBySliceFunctor::Plane::XZ) + if(extraBytesResult.value() > static_cast(std::numeric_limits::max())) + { + return MakeErrorResult(-45465, fmt::format("Identify Sample mask '{}': {} extra planes * {} bytes exceeds the allocation bound {} bytes.", maskName, maximumExtras, layout.PlaneBytes, + std::numeric_limits::max())); + } + } +#if SIMPLNX_BUILD_TESTS + IdentifySampleCarrierObservation planeObservation(maskArray, false, sizeof(T)); +#endif + auto planeBuffer = std::make_unique(layout.PlaneValues); +#if SIMPLNX_BUILD_TESTS + planeObservation.acquired(layout.PlaneValues); +#endif + + if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY || plane == IdentifySampleSliceBySliceFunctor::Plane::XZ) + { + ThrottledMessageHandler progressThrottle(messageHandler); + progressThrottle.reset(layout.FixedCount, "Identifying Sample Slices"); + for(usize fixed = 0; fixed < layout.FixedCount; ++fixed) { - for(int64 z = 0; z < dimZ; z++) + if(shouldCancel) + { + return {}; + } + if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY) { - auto readResult = store.copyIntoBuffer(static_cast(z * dimX * dimY + fixed * dimX), nonstd::span(planeBuffer.get() + static_cast(z * dimX), static_cast(dimX))); + auto readResult = NX_IDENTIFY_SAMPLE_SLICE_READ(maskArray, store, fixed * layout.PlaneValues, nonstd::span(planeBuffer.get(), layout.PlaneValues)); if(readResult.invalid()) { return readResult; } } - } - else - { - for(int64 z = 0; z < dimZ; z++) + else { - auto readResult = store.copyIntoBuffer(static_cast(z) * zSliceSize, nonstd::span(zBuffer.get(), zSliceSize)); - if(readResult.invalid()) + for(usize z = 0; z < layout.Z; ++z) { - return readResult; + auto readResult = NX_IDENTIFY_SAMPLE_SLICE_READ(maskArray, store, (z * layout.XyValues) + (fixed * layout.X), nonstd::span(planeBuffer.get() + (z * layout.X), layout.X)); + if(readResult.invalid()) + { + return readResult; + } } - for(int64 y = 0; y < dimY; y++) + } + auto identifyResult = identifyPlane(planeBuffer.get(), layout.PlaneDim1, layout.PlaneDim2, fillHoles, allowInMemoryFallback, shouldCancel); + if(identifyResult.invalid()) + { + return identifyResult; + } + if(shouldCancel) + { + return {}; + } + if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY) + { + auto writeResult = NX_IDENTIFY_SAMPLE_SLICE_WRITE(maskArray, store, fixed * layout.PlaneValues, nonstd::span(planeBuffer.get(), layout.PlaneValues)); + if(writeResult.invalid()) + { + return writeResult; + } + } + else + { + for(usize z = 0; z < layout.Z; ++z) { - planeBuffer[static_cast(z * dimY + y)] = zBuffer[static_cast(y * dimX + fixed)]; + auto writeResult = NX_IDENTIFY_SAMPLE_SLICE_WRITE(maskArray, store, (z * layout.XyValues) + (fixed * layout.X), nonstd::span(planeBuffer.get() + (z * layout.X), layout.X)); + if(writeResult.invalid()) + { + return writeResult; + } } } + progressThrottle.updateCount(fixed + 1); } + return {}; + } - auto identifyResult = identifyPlane(planeBuffer.get(), planeDim1, planeDim2, fillHoles, allowInMemoryFallback, shouldCancel); - if(identifyResult.invalid()) - { - return identifyResult; - } +#if SIMPLNX_BUILD_TESTS + IdentifySampleCarrierObservation xyObservation(maskArray, true, sizeof(T)); +#endif + auto zBuffer = std::make_unique(layout.XyValues); +#if SIMPLNX_BUILD_TESTS + xyObservation.acquired(layout.XyValues); +#endif + auto& budget = CacheMemoryBudgetManager::instance(); + // The first write to an out-of-core mask can need its own working reservation, for example to create a + // writable copy of a read-only imported source. Publish the first batch without extra planes so that this + // reservation never competes with them, then group later batches. + bool extraPlanesAllowed = store.getStoreType() != IDataStore::StoreType::OutOfCore; + for(usize batchStart = 0; batchStart < layout.FixedCount;) + { if(shouldCancel) { return {}; } - - if(plane == IdentifySampleSliceBySliceFunctor::Plane::XY) + const usize remaining = layout.FixedCount - batchStart; + const usize maxExtra = layout.PlaneBytes == 0 || !extraPlanesAllowed ? 0 : std::min(maxYzBatchPlanes - 1, remaining - 1); + // The maximum request was checked before any carrier allocation. + const usize requestedBytes = maxExtra * layout.PlaneBytes; +#if SIMPLNX_BUILD_TESTS + usize retainedBytes = 0; + bool reservationConstructed = false; + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::ExtraBytesRequested, requestedBytes, 1, true); + // Admission can throw before a token exists. Report only an actual token's destruction. + const auto reservationObservation = MakeScopeGuard([&]() noexcept { + if(reservationConstructed) + { + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::ExtraReservationReleased, retainedBytes, 1, true); + } + }); +#endif + auto reservation = budget.reserveWorkingMemory(static_cast(requestedBytes)); +#if SIMPLNX_BUILD_TESTS + reservationConstructed = true; +#endif + const uint64 grantedBytes = reservation.sizeBytes(); +#if SIMPLNX_BUILD_TESTS + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::ExtraBytesGranted, static_cast(grantedBytes), 1, true); +#endif + const usize extras = layout.PlaneBytes == 0 ? 0 : static_cast(std::min(maxExtra, grantedBytes / layout.PlaneBytes)); + // The checked request bounds both products because extras cannot exceed maxExtra. + const usize allocatedBytes = extras * layout.PlaneBytes; + const usize extraValues = extras * layout.PlaneValues; + reservation.shrinkTo(static_cast(allocatedBytes)); +#if SIMPLNX_BUILD_TESTS + retainedBytes = allocatedBytes; + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::ExtraBytesRetained, allocatedBytes, 1, true); + usize ownedExtraValues = 0; + // Values die before their release event, and the token outlives both. + const auto extraObservation = MakeScopeGuard([&]() noexcept { + if(ownedExtraValues != 0) + { + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::ExtraPlanesReleased, ownedExtraValues, sizeof(T), true); + } + }); +#endif + std::unique_ptr extraPlanes; + if(extras != 0) { - auto writeResult = store.copyFromBuffer(static_cast(fixed) * planeSize, nonstd::span(planeBuffer.get(), planeSize)); - if(writeResult.invalid()) +#if SIMPLNX_BUILD_TESTS + if(IdentifySampleExtraAllocationControl.beforeAllocate != nullptr) { - return writeResult; + IdentifySampleExtraAllocationControl.beforeAllocate(IdentifySampleExtraAllocationControl.context, maskArray, extraValues); } +#endif + extraPlanes = std::make_unique(extraValues); +#if SIMPLNX_BUILD_TESTS + ownedExtraValues = extraValues; + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::ExtraPlanesAllocated, extraValues, sizeof(T), true); +#endif } - else if(plane == IdentifySampleSliceBySliceFunctor::Plane::XZ) + std::array planePointers{}; + planePointers[0] = planeBuffer.get(); + for(usize p = 1; p <= extras; ++p) { - for(int64 z = 0; z < dimZ; z++) + planePointers[p] = extraPlanes.get() + (p - 1) * layout.PlaneValues; + } + const usize width = 1 + extras; +#if SIMPLNX_BUILD_TESTS + ObserveIdentifySampleSlice(maskArray, IdentifySampleSliceEventForTesting::BatchWidth, width, 1, true); +#endif + for(usize z = 0; z < layout.Z; ++z) + { + if(shouldCancel) { - auto writeResult = - store.copyFromBuffer(static_cast(z * dimX * dimY + fixed * dimX), nonstd::span(planeBuffer.get() + static_cast(z * dimX), static_cast(dimX))); - if(writeResult.invalid()) + return {}; + } + auto readResult = NX_IDENTIFY_SAMPLE_SLICE_READ(maskArray, store, z * layout.XyValues, nonstd::span(zBuffer.get(), layout.XyValues)); + if(readResult.invalid()) + { + return readResult; + } + for(usize y = 0; y < layout.Y; ++y) + { + for(usize p = 0; p < width; ++p) { - return writeResult; + planePointers[p][(z * layout.Y) + y] = zBuffer[(y * layout.X) + batchStart + p]; } } } - else + + usize completedPlaneCount = 0; + Result<> primaryResult; + std::exception_ptr primaryException; + DeferredSliceFailureLocation primaryLocation; + bool terminal = false; + for(usize p = 0; p < width; ++p) { - for(int64 z = 0; z < dimZ; z++) + if(shouldCancel) + { + terminal = true; + break; + } + primaryLocation = {DeferredSlicePhase::ReportPlane, batchStart + p, batchStart, completedPlaneCount, std::nullopt}; + try + { + // Keep this callback inside the guard so a failure still publishes the completed prefix. + messageHandler.sendProgressCount("Identifying Sample Slices", batchStart + p, layout.FixedCount); + primaryLocation.Phase = DeferredSlicePhase::ClassifyPlane; + auto planeResult = identifyPlane(planePointers[p], layout.PlaneDim1, layout.PlaneDim2, fillHoles, allowInMemoryFallback, shouldCancel); + if(planeResult.invalid()) + { + primaryResult = std::move(planeResult); + terminal = true; + break; + } + if(shouldCancel) + { + terminal = true; + break; + } + ++completedPlaneCount; + } catch(...) { - auto readResult = store.copyIntoBuffer(static_cast(z) * zSliceSize, nonstd::span(zBuffer.get(), zSliceSize)); + primaryException = std::current_exception(); + terminal = true; + break; + } + } + + Result<> commitResult; + std::exception_ptr commitException; + DeferredSliceFailureLocation commitLocation; + // A completed prefix must finish its publication attempt even after cancellation. + for(usize z = 0; completedPlaneCount != 0 && z < layout.Z; ++z) + { + commitLocation = {DeferredSlicePhase::ReadPrefix, 0, batchStart, completedPlaneCount, z}; + try + { + auto readResult = NX_IDENTIFY_SAMPLE_SLICE_READ(maskArray, store, z * layout.XyValues, nonstd::span(zBuffer.get(), layout.XyValues)); if(readResult.invalid()) { - return readResult; + commitResult = std::move(readResult); + break; } - for(int64 y = 0; y < dimY; y++) + for(usize y = 0; y < layout.Y; ++y) { - zBuffer[static_cast(y * dimX + fixed)] = planeBuffer[static_cast(z * dimY + y)]; + for(usize p = 0; p < completedPlaneCount; ++p) + { + zBuffer[(y * layout.X) + batchStart + p] = planePointers[p][(z * layout.Y) + y]; + } } - auto writeResult = store.copyFromBuffer(static_cast(z) * zSliceSize, nonstd::span(zBuffer.get(), zSliceSize)); + commitLocation.Phase = DeferredSlicePhase::WritePrefix; + auto writeResult = NX_IDENTIFY_SAMPLE_SLICE_WRITE(maskArray, store, z * layout.XyValues, nonstd::span(zBuffer.get(), layout.XyValues)); if(writeResult.invalid()) { - return writeResult; + commitResult = std::move(writeResult); + break; + } + } catch(...) + { + commitException = std::current_exception(); + break; + } + } + + const bool primaryFailed = primaryException != nullptr || primaryResult.invalid(); + const bool commitFailed = commitException != nullptr || commitResult.invalid(); + if(primaryFailed && commitFailed) + { + try + { +#if SIMPLNX_BUILD_TESTS + if(IdentifySampleDiagnosticControl.beforeAggregate != nullptr) + { + IdentifySampleDiagnosticControl.beforeAggregate(IdentifySampleDiagnosticControl.context, maskArray); + } +#endif + // Preserve both original carriers until all diagnostic allocation succeeds. + Result<> first = primaryException ? MakeDeferredSliceExceptionResult(primaryException, maskName, primaryLocation) : primaryResult; + Result<> second = commitException ? MakeDeferredSliceExceptionResult(commitException, maskName, commitLocation) : commitResult; + return MergeResults(std::move(first), std::move(second)); + } catch(...) + { + if(primaryException) + { + std::rethrow_exception(primaryException); } + return primaryResult; } } - progressThrottle.updateCount(static_cast(fixed) + 1); + if(primaryException) + { + std::rethrow_exception(primaryException); + } + if(primaryResult.invalid()) + { + return primaryResult; + } + if(commitException) + { + std::rethrow_exception(commitException); + } + if(commitResult.invalid()) + { + return commitResult; + } + if(terminal || shouldCancel) + { + return {}; + } + // A completed batch published its planes, so the first mask write has finished. + extraPlanesAllowed = true; + batchStart += width; } + messageHandler.sendProgressCount("Identifying Sample Slices", layout.FixedCount, layout.FixedCount); return {}; } }; @@ -1053,3 +1618,6 @@ Result<> IdentifySampleCCL::operator()() return {}; } + +#undef NX_IDENTIFY_SAMPLE_SLICE_READ +#undef NX_IDENTIFY_SAMPLE_SLICE_WRITE diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.hpp index 58f553a97d..ace5d4b267 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/IdentifySampleCCL.hpp @@ -9,8 +9,94 @@ namespace nx::core { +class IDataArray; struct IdentifySampleInputValues; +#if SIMPLNX_BUILD_TESTS +/** + * @enum IdentifySampleSliceEventForTesting + * @brief Names actual slice-copy and carrier-lifetime observations. + * @note Grant, batch, and extra-lifetime events set success true. A zero-byte grant is a valid observation. + */ +enum class IdentifySampleSliceEventForTesting +{ + ReadBegin, ///< Starts one logical bulk read. + ReadEnd, ///< Ends the read, including error or exception exit. + WriteBegin, ///< Starts one logical bulk write. + WriteEnd, ///< Ends the write, including error or exception exit. + PlaneAllocated, ///< The original plane carrier has allocated its values. + PlaneReleased, ///< The original plane carrier has destroyed its values. + XyAllocated, ///< The original XY carrier has allocated its values. + XyReleased, ///< The original XY carrier has destroyed its values. + ExtraPlanesAllocated, ///< Owns extra values; values counts elements of elementBytes bytes. + ExtraPlanesReleased, ///< Destroys extra values before releasing their reservation; units match allocation. + ExtraBytesRequested, ///< Requests extra bytes; values is bytes and elementBytes is one. + ExtraBytesGranted, ///< Receives actual reserved bytes; values is bytes and elementBytes is one. + ExtraBytesRetained, ///< Retains whole-plane bytes after shrinking; values is bytes and elementBytes is one. + BatchWidth, ///< Selects an actual batch; values is planes and elementBytes is one. + ExtraReservationReleased ///< Releases the retained token after extra values; values is bytes and elementBytes is one. +}; + +/** + * @struct IdentifySampleSliceObserverForTesting + * @brief Borrows a synchronous callback on the filter execution thread. + * @note The callback must not allocate, throw, or reenter storage. Borrowed mask identity lasts through the callback. + */ +struct SIMPLNXCORE_EXPORT IdentifySampleSliceObserverForTesting +{ + void* context = nullptr; + void (*callback)(void*, const IDataArray*, IdentifySampleSliceEventForTesting, usize values, usize elementBytes, bool success) noexcept = nullptr; +}; + +/** + * @brief Replaces this thread's slice observer. + * @param observer Supplies the callback and its borrowed context. + * @return Previous observer for scoped restoration. + * @pre No observed call or carrier owner is live on this thread. + */ +SIMPLNXCORE_EXPORT IdentifySampleSliceObserverForTesting SetIdentifySampleSliceObserverForTesting(IdentifySampleSliceObserverForTesting observer) noexcept; + +/** + * @struct IdentifySampleExtraAllocationControlForTesting + * @brief Borrows a fault callback immediately before a nonzero extra-plane allocation. + * @note The callback can throw. It does not change admission or report successful allocation. + * @note The borrowed context and mask identity remain valid through the synchronous callback. + */ +struct SIMPLNXCORE_EXPORT IdentifySampleExtraAllocationControlForTesting +{ + void* context = nullptr; + void (*beforeAllocate)(void*, const IDataArray*, usize extraValues) = nullptr; +}; + +/** + * @brief Replaces this thread's extra-allocation control. + * @param control Supplies the callback and borrowed context. + * @return Previous control for scoped restoration. + * @pre No slice execution or extra owner is live on this thread. + */ +SIMPLNXCORE_EXPORT IdentifySampleExtraAllocationControlForTesting SetIdentifySampleExtraAllocationControlForTesting(IdentifySampleExtraAllocationControlForTesting control) noexcept; + +/** + * @struct IdentifySampleDiagnosticControlForTesting + * @brief Borrows a fault callback before guarded dual-failure diagnostic construction. + * @note The callback can throw. Both original failure carriers remain intact at this boundary. + * @note The borrowed context and mask identity remain valid through the synchronous callback. + */ +struct SIMPLNXCORE_EXPORT IdentifySampleDiagnosticControlForTesting +{ + void* context = nullptr; + void (*beforeAggregate)(void*, const IDataArray*) = nullptr; +}; + +/** + * @brief Replaces this thread's diagnostic control. + * @param control Supplies the callback and borrowed context. + * @return Previous control for scoped restoration. + * @pre No slice execution is live on this thread. + */ +SIMPLNXCORE_EXPORT IdentifySampleDiagnosticControlForTesting SetIdentifySampleDiagnosticControlForTesting(IdentifySampleDiagnosticControlForTesting control) noexcept; +#endif + /** * @class IdentifySampleCCL * @brief Uses scanline connected-component labeling (CCL) for sequential access. @@ -27,7 +113,9 @@ struct IdentifySampleInputValues; * * Equal-sized components favor the largest provisional root label. Slice mode * uses a separate row-streaming CCL implementation. It keeps one plane buffer, - * one Z-slice buffer, two label rows, and external equivalence records. + * two label rows, and external equivalence records. YZ mode also keeps one + * Z-slice buffer and admits up to seven extra planes through a working-memory + * reservation. Only completed planes are published after a batch stops. * * Cancellation can stop between scan or replay units and return success. The * operation does not restore slices that a prior replay changed. Bulk-I/O and diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.cpp index f0ddbda129..0654d5f4fc 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.cpp @@ -2,13 +2,562 @@ #include "LaplacianSmoothing.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" +#include "simplnx/DataStructure/Geometry/INodeGeometry1D.hpp" #include "simplnx/DataStructure/Geometry/INodeGeometry2D.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/DataStructure/IO/Generic/ITemporaryRecordStore.hpp" +#include "simplnx/Filter/IFilter.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" +#include "simplnx/Utilities/Meshing/VertexUtilities.hpp" #include "simplnx/Utilities/ProgressEstimator.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + using namespace nx::core; +namespace +{ +using SmoothingIndexType = IGeometry::MeshIndexType; + +/** @brief Owns only the newly added mesh copies and their admitted working bytes. */ +struct SmoothingWorkspace +{ + CacheMemoryBudgetManager::WorkingMemoryReservation Reservation; + std::vector Vertices; + std::vector Edges; + std::vector Order; + std::vector Gathered; + std::vector Page; + std::unique_ptr Temporary; + usize EdgeCapacity = 0; + usize PublicationCapacity = 0; + bool CacheVertices = false; + bool CacheEdges = false; +}; + +/** + * @brief Releases compute buffers before their memory reservation. + * @param work Owns compute buffers, the reservation, and temporary coordinate records. + * @note Retains temporary coordinate records for publication. + */ +void releaseSmoothingCompute(SmoothingWorkspace& work) +{ + std::vector().swap(work.Vertices); + std::vector().swap(work.Edges); + std::vector().swap(work.Order); + std::vector().swap(work.Gathered); + std::vector().swap(work.Page); + work.Reservation = {}; +} + +float32 smoothingLambda(int8 nodeType, const LaplacianSmoothingInputValues& input) +{ + switch(nodeType) + { + case NodeType::Default: + return input.pLambda; + case NodeType::TriplePoint: + return input.pTripleLineLambda; + case NodeType::QuadPoint: + return input.pQuadPointLambda; + case NodeType::SurfaceDefault: + return input.pSurfacePointLambda; + case NodeType::SurfaceTriplePoint: + return input.pSurfaceTripleLineLambda; + case NodeType::SurfaceQuadPoint: + return input.pSurfaceQuadPointLambda; + default: + return 0.0F; + } +} + +Result<> readSmoothingLambdas(const Int8AbstractDataStore& nodes, std::vector& lambdas, const LaplacianSmoothingInputValues& input) +{ + const usize count = nodes.getNumberOfTuples(); + lambdas.assign(count, 0.0F); + if(count == 0) + { + return {}; + } + auto& budget = CacheMemoryBudgetManager::instance(); + auto reservation = budget.reserveWorkingMemory(std::min(count, budget.maximumWorkingMemoryBytes())); + const usize capacity = static_cast(std::min(count, reservation.sizeBytes() / sizeof(int8))); + if(capacity == 0) + { + return MakeErrorResult(-564, fmt::format("Smoothing node types at '{}' requested {} working bytes and received {}. At least {} bytes are required.", input.pSurfaceMeshNodeTypeArrayPath.toString(), + count, reservation.sizeBytes(), sizeof(int8))); + } + std::vector page(capacity); + Result<> result; + for(usize first = 0; first < count;) + { + const usize size = std::min(capacity, count - first); + result = MergeResults(std::move(result), nodes.copyIntoBuffer(first, {page.data(), size})); + if(result.invalid()) + { + return MergeResults(std::move(result), + MakeErrorResult(-560, fmt::format("Error reading node types at path '{}' (first vertex {}, count {}).", input.pSurfaceMeshNodeTypeArrayPath.toString(), first, size))); + } + for(usize index = 0; index < size; ++index) + { + lambdas[first + index] = smoothingLambda(page[index], input); + } + first += size; + } + return result; +} + +Result<> readTemporaryCoordinates(const ITemporaryRecordStore& temporary, usize first, nonstd::span page) +{ + const std::atomic_bool active{false}; + const usize count = page.size() / 3; + const auto bytes = std::as_writable_bytes(std::span(page.data(), page.size())); + auto read = temporary.read(first, count, {bytes.data(), bytes.size()}, active); + if(read.invalid()) + { + return ConvertResult(std::move(read)); + } + if(read.value() != count) + { + return MergeResults(ConvertResult(std::move(read)), + MakeErrorResult(-565, fmt::format("Smoothing temporary coordinates returned a short read at vertex {}: expected {} XYZ records.", first, count))); + } + return ConvertResult(std::move(read)); +} + +Result<> writeTemporaryCoordinates(ITemporaryRecordStore& temporary, usize first, nonstd::span page) +{ + const std::atomic_bool active{false}; + const auto bytes = std::as_bytes(std::span(page.data(), page.size())); + return temporary.write(first, page.size() / 3, {bytes.data(), bytes.size()}, active); +} + +Result<> prepareSmoothingWorkspace(SmoothingWorkspace& work, const Float32AbstractDataStore& vertices, const AbstractDataStore& edges) +{ + constexpr usize maximum = std::numeric_limits::max(); + constexpr usize vertexUnit = 3 * sizeof(float32); + constexpr usize edgeUnit = 2 * sizeof(SmoothingIndexType); + constexpr usize gatherUnit = edgeUnit + (2 * sizeof(usize)) + (12 * sizeof(float32)); + const usize vertexCount = vertices.getNumberOfTuples(); + const usize edgeCount = edges.getNumberOfTuples(); + if(vertexCount > maximum / vertexUnit || edgeCount > maximum / edgeUnit || vertexCount * vertexUnit > maximum - edgeCount * edgeUnit) + { + return MakeErrorResult(-566, "Smoothing coordinates and edges exceed the addressable buffer size. Reduce the mesh size."); + } + const usize vertexBytes = vertexCount * vertexUnit; + const usize edgeBytes = edgeCount * edgeUnit; + const usize requested = vertexBytes + edgeBytes; + auto& budget = CacheMemoryBudgetManager::instance(); + work.Reservation = budget.reserveWorkingMemory(std::min(requested, budget.maximumWorkingMemoryBytes())); + const usize granted = static_cast(std::min(work.Reservation.sizeBytes(), maximum)); + const usize gatherEdges = std::min(std::max(usize{1}, edgeCount), granted / gatherUnit); + work.CacheEdges = vertexBytes <= granted && edgeBytes <= granted - vertexBytes; + // A coordinate-only shortcut must retain the useful edge capacity of the fallback. + work.CacheVertices = work.CacheEdges || (vertexBytes <= granted && std::max(usize{1}, gatherEdges) <= (granted - vertexBytes) / edgeUnit); + work.EdgeCapacity = gatherEdges; + if(work.CacheEdges) + { + work.EdgeCapacity = std::max(usize{1}, edgeCount); + } + else if(work.CacheVertices) + { + work.EdgeCapacity = std::min(std::max(usize{1}, edgeCount), (granted - vertexBytes) / edgeUnit); + } + if(work.EdgeCapacity == 0) + { + return MakeErrorResult(-567, fmt::format("Smoothing requested {} working bytes and received {}. At least {} bytes are needed for a gathered edge, or {} bytes for cached coordinates and one edge.", + requested, granted, gatherUnit, vertexBytes <= maximum - edgeUnit ? vertexBytes + edgeUnit : maximum)); + } + work.PublicationCapacity = std::min(vertexCount, std::max(usize{1}, granted / vertexUnit)); + work.Vertices.resize(work.CacheVertices ? vertexCount * 3 : 0); + work.Edges.resize((work.CacheEdges ? edgeCount : work.EdgeCapacity) * 2); + Result<> result; + if(work.CacheVertices) + { + result = MergeResults(std::move(result), vertices.copyIntoBuffer(0, {work.Vertices.data(), work.Vertices.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-561, "Error reading smoothing vertex coordinates.")); + } + } + else + { + const usize pageVertices = std::min(vertexCount, work.EdgeCapacity * 2); + work.Order.resize(work.EdgeCapacity * 2); + work.Gathered.resize(work.EdgeCapacity * 6); + work.Page.resize(pageVertices * 3); + work.PublicationCapacity = pageVertices; + TemporaryRecordStoreConfig config; + config.recordSize = vertexUnit; + config.initialRecordCount = vertexCount; + config.maxRecordsPerBatch = pageVertices; + config.readOnly = false; + auto created = DataStoreUtilities::CreateTemporaryRecordStore(config); + if(created.invalid()) + { + return MergeResults(std::move(result), ConvertResult(std::move(created))); + } + if(created.value() == nullptr) + { + return MergeResults(MergeResults(std::move(result), ConvertResult(std::move(created))), MakeErrorResult(-568, "Smoothing temporary-record provider returned a null coordinate store.")); + } + work.Temporary = std::move(created.value()); + result = MergeResults(std::move(result), ConvertResult(std::move(created))); + for(usize first = 0; first < vertexCount;) + { + const usize count = std::min(pageVertices, vertexCount - first); + result = MergeResults(std::move(result), vertices.copyIntoBuffer(first * 3, {work.Page.data(), count * 3})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-561, fmt::format("Error reading smoothing coordinates at vertex {} ({} vertices).", first, count))); + } + result = MergeResults(std::move(result), writeTemporaryCoordinates(*work.Temporary, first, {work.Page.data(), count * 3})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-569, fmt::format("Error initializing temporary smoothing coordinates at vertex {} ({} vertices).", first, count))); + } + first += count; + } + } + if(work.CacheEdges) + { + result = MergeResults(std::move(result), edges.copyIntoBuffer(0, {work.Edges.data(), work.Edges.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-562, "Error reading smoothing edges.")); + } + } + return result; +} + +Result<> accumulateSmoothingDeltas(SmoothingWorkspace& work, const AbstractDataStore& edges, usize vertexCount, std::vector& deltas, std::vector& connections) +{ + const usize edgeCount = edges.getNumberOfTuples(); + Result<> result; + for(usize first = 0; first < edgeCount;) + { + const usize count = std::min(work.EdgeCapacity, edgeCount - first); + if(!work.CacheEdges) + { + result = MergeResults(std::move(result), edges.copyIntoBuffer(first * 2, {work.Edges.data(), count * 2})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-562, fmt::format("Error reading smoothing edges at edge {} ({} edges).", first, count))); + } + } + const auto block = nonstd::span(work.Edges.data(), work.Edges.size()).subspan(work.CacheEdges ? first * 2 : 0, count * 2); + for(usize endpoint = 0; endpoint < count * 2; ++endpoint) + { + if(block[endpoint] >= vertexCount) + { + return MergeResults(std::move(result), + MakeErrorResult(-570, fmt::format("Smoothing edge {} references vertex {}, but the vertex count is {}.", first + (endpoint / 2), block[endpoint], vertexCount))); + } + } + if(!work.CacheVertices) + { + result = MergeResults(std::move(result), MeshingUtilities::detail::GatherVertexCoordinates(*work.Temporary, block, {work.Order.data(), count * 2}, {work.Gathered.data(), count * 6}, + {work.Page.data(), work.Page.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-571, fmt::format("Error gathering smoothing coordinates for edge {} ({} edges).", first, count))); + } + } + for(usize edge = 0; edge < count; ++edge) + { + const usize in1 = static_cast(block[edge * 2]); + const usize in2 = static_cast(block[(edge * 2) + 1]); + const nonstd::span coordinates = + work.CacheVertices ? nonstd::span(work.Vertices.data(), work.Vertices.size()) : nonstd::span(work.Gathered.data(), work.Gathered.size()); + const usize firstOffset = work.CacheVertices ? in1 * 3 : edge * 6; + const usize secondOffset = work.CacheVertices ? in2 * 3 : (edge * 6) + 3; + const auto firstPoint = coordinates.subspan(firstOffset, 3); + const auto secondPoint = coordinates.subspan(secondOffset, 3); + for(usize component = 0; component < 3; ++component) + { + const double delta = static_cast(secondPoint[component] - firstPoint[component]); + deltas[(in1 * 3) + component] += delta; + deltas[(in2 * 3) + component] += -1.0 * delta; + } + connections[in1] += 1; + connections[in2] += 1; + } + first += count; + } + return result; +} + +Result<> updateSmoothingCoordinates(SmoothingWorkspace& work, const std::vector& lambdas, std::vector& deltas, std::vector& connections, bool negativePass, float32 muFactor) +{ + const usize vertexCount = lambdas.size(); + const usize capacity = work.CacheVertices ? std::max(usize{1}, vertexCount) : work.Page.size() / 3; + Result<> result; + for(usize first = 0; first < vertexCount;) + { + const usize count = std::min(capacity, vertexCount - first); + const auto coordinates = work.CacheVertices ? nonstd::span(work.Vertices.data(), work.Vertices.size()).subspan(first * 3, count * 3) : + nonstd::span(work.Page.data(), work.Page.size()).first(count * 3); + if(!work.CacheVertices) + { + result = MergeResults(std::move(result), readTemporaryCoordinates(*work.Temporary, first, coordinates)); + if(result.invalid()) + { + return result; + } + } + for(usize local = 0; local < count; ++local) + { + const usize vertex = first + local; + for(usize component = 0; component < 3; ++component) + { + const usize flat = (vertex * 3) + component; + const double delta = deltas[flat] / connections[vertex]; + const float32 lambda = negativePass ? lambdas[vertex] * muFactor : lambdas[vertex]; + coordinates[(local * 3) + component] += static_cast(lambda * delta); + deltas[flat] = 0.0; + } + connections[vertex] = 0; + } + if(!work.CacheVertices) + { + result = MergeResults(std::move(result), writeTemporaryCoordinates(*work.Temporary, first, {coordinates.data(), coordinates.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-572, fmt::format("Error updating temporary smoothing coordinates at vertex {} ({} vertices).", first, count))); + } + } + first += count; + } + return result; +} + +Result<> readPublicationCoordinates(ITemporaryRecordStore* temporaryPtr, std::span staged, usize first, nonstd::span output) +{ + if(temporaryPtr != nullptr) + { + return readTemporaryCoordinates(*temporaryPtr, first, output); + } + const usize offset = first * 3 * sizeof(float32); + const usize bytes = output.size_bytes(); + if(offset > staged.size() || bytes > staged.size() - offset) + { + return MakeErrorResult(-573, "Smoothing publication exceeds the completed coordinate byte range."); + } + std::memcpy(output.data(), staged.subspan(offset, bytes).data(), bytes); + return {}; +} + +/** + * @brief Writes completed coordinates after releasing compute buffers and their reservation. + * @param work Supplies cached or temporary coordinates and the preferred publication page size. + * @param vertices Receives XYZ tuples in order; earlier writes remain if a later operation fails. + * @param terminalDelta Takes ownership of the completed delta buffer for cached-coordinate byte staging. + * @return Success with I/O warnings, or a staging, admission, read, or write error. + * @note Releasing compute admission lets the output store reserve its copy-on-write buffers. + * After byte staging, terminalDelta provides storage only. Do not read or resize its double elements. + * Publication does not check cancellation. Completed passes can still be published after cancellation. + */ +Result<> publishSmoothedCoordinates(SmoothingWorkspace& work, Float32AbstractDataStore& vertices, std::vector terminalDelta) +{ + const usize vertexCount = vertices.getNumberOfTuples(); + constexpr usize maximum = std::numeric_limits::max(); + if(vertexCount == 0) + { + releaseSmoothingCompute(work); + return {}; + } + if(vertexCount > maximum / (3 * sizeof(float32)) || terminalDelta.size() > maximum / sizeof(double)) + { + return MakeErrorResult(-574, "Completed smoothing coordinates exceed the addressable terminal byte range."); + } + const usize coordinateBytes = vertexCount * 3 * sizeof(float32); + std::span staged; + if(work.CacheVertices) + { + auto storage = std::as_writable_bytes(std::span(terminalDelta)); + if(work.Vertices.size() != vertexCount * 3 || coordinateBytes > storage.size()) + { + return MakeErrorResult(-574, "Completed smoothing coordinates do not fit the existing terminal delta storage."); + } + std::memcpy(storage.data(), work.Vertices.data(), coordinateBytes); + staged = storage.first(coordinateBytes); + } + const usize preferredCapacity = work.PublicationCapacity; + releaseSmoothingCompute(work); + auto& budget = CacheMemoryBudgetManager::instance(); + Result<> result; + { + auto firstReservation = budget.reserveWorkingMemory(3 * sizeof(float32)); + if(firstReservation.sizeBytes() < 3 * sizeof(float32)) + { + return MakeErrorResult(-575, fmt::format("Smoothing publication needs {} working bytes for its first XYZ tuple, but received {}.", 3 * sizeof(float32), firstReservation.sizeBytes())); + } + std::array first{}; + result = MergeResults(std::move(result), readPublicationCoordinates(work.Temporary.get(), staged, 0, {first.data(), first.size()})); + if(result.invalid()) + { + return result; + } + // This real first output tuple permits an imported store to reserve its own COW buffers. + result = MergeResults(std::move(result), vertices.copyFromBuffer(0, {first.data(), first.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-563, "Error writing the first smoothed XYZ tuple.")); + } + } + if(vertexCount == 1) + { + return result; + } + auto reservation = budget.reserveWorkingMemory(std::min(preferredCapacity * 3 * sizeof(float32), budget.maximumWorkingMemoryBytes())); + const usize capacity = static_cast(std::min(preferredCapacity, reservation.sizeBytes() / (3 * sizeof(float32)))); + if(capacity == 0) + { + return MergeResults(std::move(result), + MakeErrorResult(-575, fmt::format("Smoothing publication cannot admit one XYZ page after its first tuple. Received {} working bytes.", reservation.sizeBytes()))); + } + std::vector page(capacity * 3); + for(usize first = 1; first < vertexCount;) + { + const usize count = std::min(capacity, vertexCount - first); + result = MergeResults(std::move(result), readPublicationCoordinates(work.Temporary.get(), staged, first, {page.data(), count * 3})); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), vertices.copyFromBuffer(first * 3, {page.data(), count * 3})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-563, fmt::format("Error writing smoothed XYZ tuples at vertex {} ({} vertices).", first, count))); + } + first += count; + } + return result; +} + +Result<> smoothOutOfCoreMesh(DataStructure& dataStructure, const LaplacianSmoothingInputValues& input, const std::atomic_bool& shouldCancel, const IFilter::MessageHandler& messages) +{ + Result<> result; + try + { + auto& geometry = dataStructure.getDataRefAs(input.pTriangleGeometryDataPath); + auto& vertices = geometry.getVertices()->getDataStoreRef(); + auto& nodes = dataStructure.getDataRefAs(input.pSurfaceMeshNodeTypeArrayPath).getDataStoreRef(); + const usize vertexCount = vertices.getNumberOfTuples(); + constexpr usize maximum = std::numeric_limits::max(); + if(vertices.getNumberOfComponents() != 3 || nodes.getNumberOfComponents() != 1 || nodes.getNumberOfTuples() != vertexCount || vertexCount > maximum / (3 * sizeof(double))) + { + return MakeErrorResult(-576, "Smoothing requires addressable XYZ vertices and one node classification per vertex. Repair the array shapes or reduce the mesh size."); + } + std::vector lambdas; + result = MergeResults(std::move(result), readSmoothingLambdas(nodes, lambdas, input)); + if(result.invalid()) + { + return result; + } + if(auto* trianglesPtr = dataStructure.getDataAs(input.pTriangleGeometryDataPath); trianglesPtr != nullptr) + { + result = MergeResults(std::move(result), trianglesPtr->findEdges(false)); + if(result.invalid()) + { + return result; + } + } + auto* edgeArrayPtr = geometry.getEdges(); + if(edgeArrayPtr == nullptr || edgeArrayPtr->getNumberOfComponents() != 2) + { + return MergeResults(std::move(result), MakeErrorResult(-577, "Smoothing requires a two-component edge array. Supply or generate valid mesh edges.")); + } + auto& edges = edgeArrayPtr->getDataStoreRef(); + SmoothingWorkspace work; + result = MergeResults(std::move(result), prepareSmoothingWorkspace(work, vertices, edges)); + if(result.invalid()) + { + return result; + } + // These older arrays remain about 32 bytes per vertex with lambdas. This cache correction does not bound the complete smoother. + std::vector connections(vertexCount, 0); + std::vector deltas(vertexCount * 3, 0.0); + bool modified = false; + ProgressEstimator progressEstimator; + progressEstimator.restart(); + const usize iterationCount = input.pIterationSteps > 0 ? static_cast(input.pIterationSteps) : 0; + messages.sendProgressCount("Smoothing Iterations", 0, iterationCount); + for(int32 iteration = 0; iteration < input.pIterationSteps; ++iteration) + { + if(shouldCancel) + { + break; + } + result = MergeResults(std::move(result), accumulateSmoothingDeltas(work, edges, vertexCount, deltas, connections)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), updateSmoothingCoordinates(work, lambdas, deltas, connections, false, input.pMuFactor)); + if(result.invalid()) + { + return result; + } + modified = true; + if(input.pUseTaubinSmoothing) + { + messages.sendProgressCount("Smoothing Iterations", static_cast(iteration), iterationCount, "Starting Taubin pass"); + if(shouldCancel) + { + break; + } + result = MergeResults(std::move(result), accumulateSmoothingDeltas(work, edges, vertexCount, deltas, connections)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), updateSmoothingCoordinates(work, lambdas, deltas, connections, true, input.pMuFactor)); + if(result.invalid()) + { + return result; + } + } + messages.sendProgressCount("Smoothing Iterations", static_cast(iteration) + 1, iterationCount, progressEstimator.estimate(static_cast(iteration) + 1, iterationCount)); + } + if(modified) + { + result = MergeResults(std::move(result), publishSmoothedCoordinates(work, vertices, std::move(deltas))); + } + return result; + } catch(const std::bad_alloc&) + { + return MergeResults(std::move(result), MakeErrorResult(-578, fmt::format("Smoothing could not allocate its working buffers for '{}'. Free memory or reduce the mesh size.", + input.pTriangleGeometryDataPath.toString()))); + } catch(const std::length_error&) + { + return MergeResults(std::move(result), + MakeErrorResult(-578, fmt::format("Smoothing exceeded an addressable buffer size for '{}'. Reduce the mesh size.", input.pTriangleGeometryDataPath.toString()))); + } +} +} // namespace + LaplacianSmoothing::LaplacianSmoothing(DataStructure& dataStructure, LaplacianSmoothingInputValues* inputValues, const std::atomic_bool& shouldCancel, const IFilter::MessageHandler& mesgHandler) : m_DataStructure(dataStructure) , m_InputValues(inputValues) @@ -52,21 +601,67 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() Float32AbstractDataStore& vertDataStoreRef = nodeGeom1DRef.getVertices()->getDataStoreRef(); IGeometry::MeshIndexType numberOfVertices = nodeGeom1DRef.getNumberOfVertices(); + const auto* surfacePtr = m_DataStructure.getDataAs(m_InputValues->pTriangleGeometryDataPath); + const bool usesOutOfCoreStore = AnyOutOfCore({nodeGeom1DRef.getVertices(), nodeGeom1DRef.getEdges(), surfacePtr == nullptr ? nullptr : surfacePtr->getFaces(), + m_DataStructure.getDataAs(m_InputValues->pSurfaceMeshNodeTypeArrayPath)}); + const bool useOutOfCoreAlgorithm = ShouldUseOutOfCoreAlgorithm(usesOutOfCoreStore); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + return smoothOutOfCoreMesh(m_DataStructure, *m_InputValues, m_ShouldCancel, m_MessageHandler); + } + // Generate the Lambda Array - std::vector lambdas = generateLambdaArray(); + std::vector lambdas; + Result<> result = generateLambdaArray(lambdas); + if(result.invalid()) + { + return result; + } auto inode2DPtr = m_DataStructure.getDataAs(m_InputValues->pTriangleGeometryDataPath); if(nullptr != inode2DPtr) { // Generate the Unique Edges - if(Result<> result = inode2DPtr->findEdges(false); result.invalid()) + result = MergeResults(std::move(result), inode2DPtr->findEdges(false)); + if(result.invalid()) { return result; } } - AbstractDataStore& edges = nodeGeom1DRef.getEdges()->getDataStoreRef(); - IGeometry::MeshIndexType numEdges = edges.getNumberOfTuples(); + const AbstractDataStore& edgeDataStore = nodeGeom1DRef.getEdges()->getDataStoreRef(); + const IGeometry::MeshIndexType numEdges = edgeDataStore.getNumberOfTuples(); + + // Cache coordinates and edges once to avoid per-element OOC I/O in every iteration. + // The edge list adds 2 * numEdges * sizeof(MeshIndexType) bytes; scratch memory remains O(mesh size), like the existing vertex arrays. + std::vector vertices(numberOfVertices * 3); + result = MergeResults(std::move(result), vertDataStoreRef.copyIntoBuffer(0, nonstd::span(vertices.data(), vertices.size()))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-561, fmt::format("Error reading vertices for geometry at path '{}'.", m_InputValues->pTriangleGeometryDataPath.toString()))); + } + + std::vector edges(numEdges * 2); + result = MergeResults(std::move(result), edgeDataStore.copyIntoBuffer(0, nonstd::span(edges.data(), edges.size()))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-562, fmt::format("Error reading edges for geometry at path '{}'.", m_InputValues->pTriangleGeometryDataPath.toString()))); + } + + bool verticesModified = false; + // Persist completed passes on cancellation, at the same checkpoints as the original algorithm. + const auto writeVertices = [&]() -> Result<> { + if(verticesModified) + { + result = MergeResults(std::move(result), vertDataStoreRef.copyFromBuffer(0, nonstd::span(vertices.data(), vertices.size()))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-563, fmt::format("Error writing smoothed vertices for geometry at path '{}'.", m_InputValues->pTriangleGeometryDataPath.toString()))); + } + } + return std::move(result); + }; std::vector numConnections(numberOfVertices, 0); @@ -81,7 +676,7 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() { if(m_ShouldCancel) { - return {}; + return writeVertices(); } // Compute the Deltas for each point for(IGeometry::MeshIndexType i = 0; i < numEdges; i++) @@ -92,12 +687,12 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() for(IGeometry::MeshIndexType j = 0; j < 3; j++) { #if 0 - Q_ASSERT(static_cast(3 * in1 + j) < static_cast(numberOfVertices * 3)); - Q_ASSERT(static_cast(3 * in2 + j) < static_cast(numberOfVertices * 3)); + Q_ASSERT(static_cast((3 * in1) + j) < static_cast(numberOfVertices * 3)); + Q_ASSERT(static_cast((3 * in2) + j) < static_cast(numberOfVertices * 3)); #endif - dlta = static_cast(vertDataStoreRef[3 * in2 + j] - vertDataStoreRef[3 * in1 + j]); - deltaArray[3 * in1 + j] += dlta; - deltaArray[3 * in2 + j] += -1.0 * dlta; + dlta = static_cast(vertices[(3 * in2) + j] - vertices[(3 * in1) + j]); + deltaArray[(3 * in1) + j] += dlta; + deltaArray[(3 * in2) + j] += -1.0 * dlta; } numConnections[in1] += 1; numConnections[in2] += 1; @@ -108,25 +703,27 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() { for(IGeometry::MeshIndexType j = 0; j < 3; j++) { - IGeometry::MeshIndexType in0 = 3 * i + j; + const IGeometry::MeshIndexType in0 = (3 * i) + j; dlta = deltaArray[in0] / numConnections[i]; float ll = lambdas[i]; - vertDataStoreRef[3 * i + j] += ll * dlta; + // ValueProxy::operator+= converts the displacement to float before the addition. + vertices[(3 * i) + j] += static_cast(ll * dlta); deltaArray[in0] = 0.0; // reset for next iteration } numConnections[i] = 0; // reset for next iteration } + verticesModified = true; // Now optionally apply a negative lambda based on the mu Factor value. // This is from Taubin's paper on smoothing without shrinkage. This effectively // runs a low pass filter on the data if(m_InputValues->pUseTaubinSmoothing) { - + m_MessageHandler.sendProgressCount("Smoothing Iterations", static_cast(q), iterationCount, "Starting Taubin pass"); if(m_ShouldCancel) { - return {}; + return writeVertices(); } // Compute the Delta's for(IGeometry::MeshIndexType i = 0; i < numEdges; i++) @@ -137,12 +734,12 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() for(int32_t j = 0; j < 3; j++) { #if 0 - Q_ASSERT(static_cast(3 * in1 + j) < static_cast(numberOfVertices * 3)); - Q_ASSERT(static_cast(3 * in2 + j) < static_cast(numberOfVertices * 3)); + Q_ASSERT(static_cast((3 * in1) + j) < static_cast(numberOfVertices * 3)); + Q_ASSERT(static_cast((3 * in2) + j) < static_cast(numberOfVertices * 3)); #endif - dlta = vertDataStoreRef[3 * in2 + j] - vertDataStoreRef[3 * in1 + j]; - deltaArray[3 * in1 + j] += dlta; - deltaArray[3 * in2 + j] += -1.0 * dlta; + dlta = vertices[(3 * in2) + j] - vertices[(3 * in1) + j]; + deltaArray[(3 * in1) + j] += dlta; + deltaArray[(3 * in2) + j] += -1.0 * dlta; } numConnections[in1] += 1; numConnections[in2] += 1; @@ -153,11 +750,11 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() { for(IGeometry::MeshIndexType j = 0; j < 3; j++) { - IGeometry::MeshIndexType in0 = 3 * i + j; + const IGeometry::MeshIndexType in0 = (3 * i) + j; dlta = deltaArray[in0] / numConnections[i]; float ll = lambdas[i] * m_InputValues->pMuFactor; - vertDataStoreRef[3 * i + j] += ll * dlta; + vertices[(3 * i) + j] += static_cast(ll * dlta); deltaArray[in0] = 0.0; // reset for next iteration } numConnections[i] = 0; // reset for next iteration @@ -166,21 +763,28 @@ Result<> LaplacianSmoothing::edgeBasedSmoothing() m_MessageHandler.sendProgressCount("Smoothing Iterations", static_cast(q) + 1, iterationCount, progressEstimator.estimate(static_cast(q) + 1, iterationCount)); } - return {}; + return writeVertices(); } // ----------------------------------------------------------------------------- -std::vector LaplacianSmoothing::generateLambdaArray() const +Result<> LaplacianSmoothing::generateLambdaArray(std::vector& lambdas) const { auto& surfaceMeshNode = m_DataStructure.getDataAs(m_InputValues->pSurfaceMeshNodeTypeArrayPath)->getDataStoreRef(); size_t numNodes = surfaceMeshNode.getNumberOfTuples(); - std::vector lambdas(numNodes, 0.0f); + std::vector nodeTypes(numNodes); + Result<> result = surfaceMeshNode.copyIntoBuffer(0, nonstd::span(nodeTypes.data(), nodeTypes.size())); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-560, fmt::format("Error reading node types at path '{}'.", m_InputValues->pSurfaceMeshNodeTypeArrayPath.toString()))); + } + + lambdas.assign(numNodes, 0.0f); for(size_t i = 0; i < numNodes; ++i) { - switch(surfaceMeshNode[i]) + switch(nodeTypes[i]) { case nx::core::NodeType::Unused: break; @@ -207,7 +811,7 @@ std::vector LaplacianSmoothing::generateLambdaArray() const } } - return lambdas; + return result; } // ----------------------------------------------------------------------------- diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.hpp index b3e93eca34..b32bf30319 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/LaplacianSmoothing.hpp @@ -48,7 +48,7 @@ class SIMPLNXCORE_EXPORT LaplacianSmoothing const std::atomic_bool& m_ShouldCancel; const IFilter::MessageHandler& m_MessageHandler; - std::vector generateLambdaArray() const; + Result<> generateLambdaArray(std::vector& lambdas) const; Result<> edgeBasedSmoothing(); }; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/SharedFeatureFace.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/SharedFeatureFace.cpp index 666a0ae7b6..7f228b8333 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/SharedFeatureFace.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/SharedFeatureFace.cpp @@ -1,10 +1,22 @@ #include "SharedFeatureFace.hpp" +#include "simplnx/Common/Aliases.hpp" +#include "simplnx/Common/Range.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataArrayUtilities.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" +#include +#include +#include + +#include +#include #include #include #include @@ -44,7 +56,7 @@ class SharedFeatureFaceImpl m_SurfaceMeshFeatureFaceLabels[2 * i + 1] = faceLabelMapEntry.second; // get feature triangle count - uint64 faceId64 = ConvertToUInt64(faceLabelMapEntry.first, faceLabelMapEntry.second); + const uint64 faceId64 = ConvertToUInt64(faceLabelMapEntry.first, faceLabelMapEntry.second); m_SurfaceMeshFeatureFaceNumTriangles[i] = m_FaceSizeMap[faceId64]; if(m_ShouldCancel) { @@ -114,6 +126,238 @@ void RandomizeFaceIds(nx::core::Int32Array& featureIds, uint64 totalFeatures, In } } +Result<> ComputeSharedFeatureFacesInBlocks(DataStructure& dataStructure, const SharedFeatureFaceInputValues& inputValues, usize totalPoints, const Int32Array& faceLabels, Int32Array& featureFaceIds, + const std::atomic_bool& shouldCancel) +{ + Result<> result; + std::map faceSizeMap; + std::map faceIdMap; + int32 index = 1; + std::vector> faceLabelVector; + faceLabelVector.emplace_back(0, 0); + + // Two labels and one ID per triangle use at most 3 MiB of transfer buffers. + constexpr usize blockTriangles = 262144; + const usize bufferTriangles = std::min(blockTriangles, totalPoints); + std::vector faceLabelsBuffer(bufferTriangles * 2); + std::vector faceIdsBuffer(bufferTriangles); + auto& faceIdsStore = featureFaceIds.getDataStoreRef(); + for(usize start = 0; start < totalPoints;) + { + const usize count = std::min(blockTriangles, totalPoints - start); + result = MergeResults(std::move(result), faceLabels.getDataStoreRef().copyIntoBuffer(start * 2, nonstd::span(faceLabelsBuffer.data(), count * 2))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12910, fmt::format("Error reading face labels for shared feature faces at triangle {} ({} triangles).", start, count))); + } + + // Keep signed label ordering, unsigned key conversion and first-encounter IDs. + for(usize t = 0; t < count; ++t) + { + uint32 high = 0; + uint32 low = 0; + const int32 fl0 = faceLabelsBuffer[t * 2]; + const int32 fl1 = faceLabelsBuffer[(t * 2) + 1]; + if(fl0 < fl1) + { + high = fl0; + low = fl1; + } + else + { + high = fl1; + low = fl0; + } + const uint64 faceId64 = ConvertToUInt64(high, low); + if(faceSizeMap.find(faceId64) == faceSizeMap.end()) + { + faceSizeMap[faceId64] = 1; + faceIdMap[faceId64] = index; + faceIdsBuffer[t] = index; + faceLabelVector.emplace_back(high, low); + ++index; + } + else + { + faceSizeMap[faceId64]++; + faceIdsBuffer[t] = faceIdMap[faceId64]; + } + } + // Every ID in this span is assigned; any suffix beyond the geometry is untouched. + result = MergeResults(std::move(result), faceIdsStore.copyFromBuffer(start, nonstd::span(faceIdsBuffer.data(), count))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12911, fmt::format("Error writing shared feature face IDs at triangle {} ({} triangles).", start, count))); + } + start += count; + } + // Match the in-core cancellation point after assigning every triangle ID. + if(shouldCancel) + { + return result; + } + + // Triangle transfers finish before metadata admission. Randomization recreates only its ID buffer below. + std::vector().swap(faceLabelsBuffer); + std::vector().swap(faceIdsBuffer); + + auto& faceFeatureAttrMat = dataStructure.getDataRefAs(inputValues.GrainBoundaryAttributeMatrixPath); + const ShapeType tDims = {static_cast(index)}; + // Preserve the in-core resize sequence and its GCC false-positive suppression. +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 15 && (__GNUC__ > 12 || (__GNUC__ == 12 && __GNUC_MINOR__ >= 4)) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif + result = MergeResults(std::move(result), faceFeatureAttrMat.resizeTuples(tDims)); +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 15 && (__GNUC__ > 12 || (__GNUC__ == 12 && __GNUC_MINOR__ >= 4)) +#pragma GCC diagnostic pop +#endif + if(result.invalid()) + { + return result; + } + + auto& featureFaceLabels = dataStructure.getDataRefAs(inputValues.FeatureFaceLabelsArrayPath).getDataStoreRef(); + auto& featureFaceNumTriangles = dataStructure.getDataRefAs(inputValues.FeatureFaceNumTrianglesArrayPath).getDataStoreRef(); + result = MergeResults(std::move(result), featureFaceLabels.resizeTuples(tDims)); + if(result.invalid()) + { + return result; + } + result = MergeResults(std::move(result), featureFaceNumTriangles.resizeTuples(tDims)); + if(result.invalid()) + { + return result; + } + + { + // Each admitted row holds both labels and its count. Cap rows before multiplying to keep byte arithmetic bounded. + constexpr usize kMetadataBytesPerRow = 3 * sizeof(int32); + constexpr usize kPreferredMetadataRows = 262144; + const usize requestedBytes = std::min(kPreferredMetadataRows, faceLabelVector.size()) * kMetadataBytesPerRow; + auto reservation = CacheMemoryBudgetManager::instance().reserveWorkingMemory(requestedBytes); + const usize blockRows = static_cast(reservation.sizeBytes() / kMetadataBytesPerRow); + if(blockRows == 0) + { + return MergeResults(std::move(result), + MakeErrorResult(-12918, fmt::format("Cannot allocate shared face metadata buffers for '{}': requested {} bytes but received {}. At least {} bytes are required. " + "Release other working memory or increase the cache budget and retry.", + inputValues.GrainBoundaryAttributeMatrixPath.toString(), requestedBytes, reservation.sizeBytes(), kMetadataBytesPerRow))); + } + reservation.shrinkTo(blockRows * kMetadataBytesPerRow); + std::vector metadataBuffer(blockRows * 3); + const nonstd::span metadataValues(metadataBuffer.data(), metadataBuffer.size()); + const auto& countsByFace = faceSizeMap; + const auto& labelsByFace = faceLabelVector; + for(usize start = 0; start < faceLabelVector.size();) + { + const usize count = std::min(blockRows, faceLabelVector.size() - start); + const auto featureLabelsBuffer = metadataValues.first(count * 2); + const auto featureCountsBuffer = metadataValues.subspan(blockRows * 2, count); + // Pre-read both arrays so cancellation preserves every row that the fill loop does not compute. + result = MergeResults(std::move(result), featureFaceLabels.copyIntoBuffer(start * 2, featureLabelsBuffer)); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12912, fmt::format("Error reading shared feature face labels '{}' at row {} ({} rows).", + inputValues.FeatureFaceLabelsArrayPath.toString(), start, count))); + } + result = MergeResults(std::move(result), featureFaceNumTriangles.copyIntoBuffer(start, featureCountsBuffer)); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12913, fmt::format("Error reading shared feature face triangle counts '{}' at row {} ({} rows).", + inputValues.FeatureFaceNumTrianglesArrayPath.toString(), start, count))); + } + + // Workers use stable resident buffers and read-only lookups. Existing cancellation permits one serial row before publication. + ParallelDataAlgorithm dataAlg; + dataAlg.setParallelizationEnabled(!shouldCancel.load()); + dataAlg.setRange(0, count); + dataAlg.execute([&labelsByFace, &countsByFace, &featureLabelsBuffer, &featureCountsBuffer, &shouldCancel, start](const Range& range) { + for(usize row = range.min(); row < range.max(); ++row) + { + const auto& labels = labelsByFace[start + row]; + featureLabelsBuffer[row * 2] = labels.first; + featureLabelsBuffer[(row * 2) + 1] = labels.second; + const auto found = countsByFace.find(ConvertToUInt64(labels.first, labels.second)); + featureCountsBuffer[row] = found == countsByFace.end() ? 0 : found->second; + if(shouldCancel) + { + break; + } + } + }); + + // Publish both arrays before honoring cancellation; earlier successful writes are not rolled back after an I/O error. + result = MergeResults(std::move(result), featureFaceLabels.copyFromBuffer(start * 2, featureLabelsBuffer)); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12914, fmt::format("Error writing shared feature face labels '{}' at row {} ({} rows).", + inputValues.FeatureFaceLabelsArrayPath.toString(), start, count))); + } + result = MergeResults(std::move(result), featureFaceNumTriangles.copyFromBuffer(start, featureCountsBuffer)); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12915, fmt::format("Error writing shared feature face triangle counts '{}' at row {} ({} rows).", + inputValues.FeatureFaceNumTrianglesArrayPath.toString(), start, count))); + } + start += count; + if(shouldCancel) + { + break; + } + } + } + + if(inputValues.ShouldRandomizeFeatureIds) + { + // Match RandomizeFaceIds' seed, int64 distribution and swap order exactly. + const uint64 totalFeatures = index; + Int64Distribution distribution; + auto generator = initializeStaticVoxelSeedGenerator(distribution, 1, totalFeatures - 1); + std::vector rndNumbers(totalFeatures); + for(int64 i = 0; i < totalFeatures; ++i) + { + rndNumbers[i] = i; + } + for(int64 i = 1; i < totalFeatures; ++i) + { + const int64 r = distribution(generator); + if(r >= totalFeatures) + { + continue; + } + const int64 temp = rndNumbers[i]; + rndNumbers[i] = rndNumbers[r]; + rndNumbers[r] = temp; + } + + // The original randomization visits the entire ID array, even if it has more + // tuples than the geometry. Read each block before applying the permutation. + const usize totalIds = featureFaceIds.getNumberOfTuples(); + faceIdsBuffer.resize(std::min(blockTriangles, totalIds)); + for(usize start = 0; start < totalIds;) + { + const usize count = std::min(blockTriangles, totalIds - start); + result = MergeResults(std::move(result), faceIdsStore.copyIntoBuffer(start, nonstd::span(faceIdsBuffer.data(), count))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12916, fmt::format("Error reading shared feature face IDs for randomization at tuple {} ({} tuples).", start, count))); + } + for(usize i = 0; i < count; ++i) + { + faceIdsBuffer[i] = rndNumbers[faceIdsBuffer[i]]; + } + result = MergeResults(std::move(result), faceIdsStore.copyFromBuffer(start, nonstd::span(faceIdsBuffer.data(), count))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-12917, fmt::format("Error writing randomized shared feature face IDs at tuple {} ({} tuples).", start, count))); + } + start += count; + } + } + return result; +} + } // namespace // ----------------------------------------------------------------------------- @@ -145,6 +389,15 @@ Result<> SharedFeatureFace::operator()() auto& surfaceMeshFeatureFaceIds = m_DataStructure.getDataRefAs(m_InputValues->FeatureFaceIdsArrayPath); + const bool usesOutOfCoreStore = AnyOutOfCore({&surfaceMeshFaceLabels, &surfaceMeshFeatureFaceIds, m_DataStructure.getDataAs(m_InputValues->FeatureFaceLabelsArrayPath), + m_DataStructure.getDataAs(m_InputValues->FeatureFaceNumTrianglesArrayPath)}); + const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + return ComputeSharedFeatureFacesInBlocks(m_DataStructure, *m_InputValues, totalPoints, surfaceMeshFaceLabels, surfaceMeshFeatureFaceIds, m_ShouldCancel); + } + std::map faceSizeMap; std::map faceIdMap; // This maps a unique 64-bit integer to an increasing 32-bit integer int32 index = 1; @@ -157,8 +410,8 @@ Result<> SharedFeatureFace::operator()() { uint32 high = 0; uint32 low = 0; - int32 fl0 = surfaceMeshFaceLabels[t * 2]; - int32 fl1 = surfaceMeshFaceLabels[t * 2 + 1]; + const int32 fl0 = surfaceMeshFaceLabels[t * 2]; + const int32 fl1 = surfaceMeshFaceLabels[(t * 2) + 1]; if(fl0 < fl1) { high = fl0; @@ -169,7 +422,7 @@ Result<> SharedFeatureFace::operator()() high = fl1; low = fl0; } - uint64 faceId64 = ConvertToUInt64(high, low); + const uint64 faceId64 = ConvertToUInt64(high, low); if(faceSizeMap.find(faceId64) == faceSizeMap.end()) { faceSizeMap[faceId64] = 1; @@ -191,7 +444,7 @@ Result<> SharedFeatureFace::operator()() // resize + update pointers // Grain Boundary Attribute Matrix auto& faceFeatureAttrMat = m_DataStructure.getDataRefAs(m_InputValues->GrainBoundaryAttributeMatrixPath); - ShapeType tDims = {static_cast(index)}; + const ShapeType tDims = {static_cast(index)}; // This line has a Wstringop-overflow warning starting in gcc 12.4 in release configuration // This is a false positive that eventually disappears in gcc 15 // Likely related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94335 diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleDihedralAngle.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleDihedralAngle.cpp index 6ef0c99df9..d50828d32d 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleDihedralAngle.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleDihedralAngle.cpp @@ -2,11 +2,28 @@ #include "simplnx/Common/Constants.hpp" #include "simplnx/Common/Range.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" +#include "simplnx/Utilities/Meshing/VertexUtilities.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" +#include +#include + #include +#include +#include +#include +#include +#include +#include +#include using namespace nx::core; @@ -62,12 +79,12 @@ class CalculateDihedralAnglesImpl } else { - std::vector dihedralAnglesVec; - dihedralAnglesVec.push_back(k_radToDeg * acos((std::fabs(vecAB.dot(vecAC)) / (magAB * magAC)))); + std::array dihedralAnglesVec{}; + dihedralAnglesVec[0] = k_radToDeg * acos((std::fabs(vecAB.dot(vecAC)) / (magAB * magAC))); // 180 - angle because AB points out of vertex and BC points into vertex, so angle is actually angle outside of triangle - dihedralAnglesVec.push_back(180.0 - (k_radToDeg * acos((std::fabs(vecAB.dot(vecBC)) / (magAB * magBC))))); - dihedralAnglesVec.push_back(k_radToDeg * acos((std::fabs(vecBC.dot(vecAC)) / (magBC * magAC)))); + dihedralAnglesVec[1] = 180.0 - (k_radToDeg * acos((std::fabs(vecAB.dot(vecBC)) / (magAB * magBC)))); + dihedralAnglesVec[2] = k_radToDeg * acos((std::fabs(vecBC.dot(vecAC)) / (magBC * magAC))); m_DihedralAngles[triangleIndex] = *std::min_element(dihedralAnglesVec.begin(), dihedralAnglesVec.end()); } @@ -89,6 +106,205 @@ class CalculateDihedralAnglesImpl Float64AbstractDataStore& m_DihedralAngles; const std::atomic_bool& m_ShouldCancel; }; + +constexpr usize k_DihedralBlockTriangles = 262144; +constexpr usize k_DihedralFastBytesPerFace = (3 * sizeof(IGeometry::MeshIndexType)) + sizeof(float64); +constexpr usize k_DihedralGatherBytesPerFace = k_DihedralFastBytesPerFace + (3 * sizeof(usize)) + (18 * sizeof(float32)); +static_assert(k_DihedralBlockTriangles <= std::numeric_limits::max() / k_DihedralGatherBytesPerFace); + +struct TriangleDihedralWorkingPlan +{ + bool CacheVertices = false; + usize FaceCapacity = 0; + usize PageVertices = 0; + usize PayloadBytes = 0; +}; + +/** + * @brief Plans coordinate and face buffers within the supplied byte limit. + * @param triangles Supplies the face count for connectivity and angle buffers. + * @param vertices Supplies the vertex count for a coordinate cache or read page. + * @param availableBytes Limits the combined buffer payload in bytes. + * @return Plan with its total payload, or zero face capacity when no face fits. + * @pre The caller checks three-component shapes and addressable coordinate and face sizes. + * @note The caller must reserve working memory and plan again from the actual grant before allocating buffers. + */ +TriangleDihedralWorkingPlan planDihedralWorkingMemory(const AbstractDataStore& triangles, const Float32AbstractDataStore& vertices, usize availableBytes) +{ + const usize maximumTriangles = std::min(k_DihedralBlockTriangles, triangles.getNumberOfTuples()); + const usize vertexCount = vertices.getNumberOfTuples(); + const usize vertexBytes = vertexCount * 3 * sizeof(float32); + const usize gatherTriangles = std::min(maximumTriangles, availableBytes / k_DihedralGatherBytesPerFace); + const usize admissionTriangles = std::max(usize{1}, gatherTriangles); + // Retaining coordinates must preserve the gathered route's available face capacity. + const bool cacheVertices = vertexBytes <= availableBytes && admissionTriangles * k_DihedralFastBytesPerFace <= availableBytes - vertexBytes; + const usize faceCapacity = cacheVertices ? std::min(maximumTriangles, (availableBytes - vertexBytes) / k_DihedralFastBytesPerFace) : gatherTriangles; + if(faceCapacity == 0) + { + return {}; + } + const usize pageVertices = cacheVertices ? 0 : std::min(vertexCount, faceCapacity * 3); + const usize coordinateBytes = cacheVertices ? vertexBytes : pageVertices * 3 * sizeof(float32); + const usize bytesPerFace = k_DihedralFastBytesPerFace + (cacheVertices ? 0 : (3 * sizeof(usize)) + (9 * sizeof(float32))); + if(coordinateBytes > availableBytes || faceCapacity > (availableBytes - coordinateBytes) / bytesPerFace) + { + return {}; + } + return {.CacheVertices = cacheVertices, .FaceCapacity = faceCapacity, .PageVertices = pageVertices, .PayloadBytes = coordinateBytes + (faceCapacity * bytesPerFace)}; +} + +Result<> CalculateDihedralAnglesInBlocks(const TriangleGeom& triangleGeom, Float64AbstractDataStore& dihedralAngles, const std::atomic_bool& shouldCancel) +{ + const auto& triangles = triangleGeom.getFaces()->getDataStoreRef(); + const auto& verts = triangleGeom.getVertices()->getDataStoreRef(); + const usize numTriangles = triangles.getNumberOfTuples(); + const usize numVertices = verts.getNumberOfTuples(); + if(shouldCancel || numTriangles == 0) + { + return {}; + } + + constexpr usize maximum = std::numeric_limits::max(); + const usize maximumTriangles = std::min(k_DihedralBlockTriangles, numTriangles); + if(verts.getNumberOfComponents() != 3 || triangles.getNumberOfComponents() != 3 || numVertices > maximum / (3 * sizeof(float32)) || numTriangles > maximum / 3 || + numVertices * (3 * sizeof(float32)) > maximum - maximumTriangles * k_DihedralFastBytesPerFace) + { + return MakeErrorResult(-65824, "Triangle dihedral-angle buffers require addressable three-component vertex and face arrays. Reduce the mesh size or repair the array shapes."); + } + const usize vertexBytes = numVertices * 3 * sizeof(float32); + const usize desiredBytes = vertexBytes + (maximumTriangles * k_DihedralFastBytesPerFace); + auto& budget = CacheMemoryBudgetManager::instance(); + const uint64 maximumWorkingBytes = budget.maximumWorkingMemoryBytes(); + const uint64 reservedBytes = budget.reservedWorkingMemoryBytes(); + const usize observedHeadroom = static_cast(std::min(maximumWorkingBytes > reservedBytes ? maximumWorkingBytes - reservedBytes : 0, maximum)); + const auto observedPlan = planDihedralWorkingMemory(triangles, verts, observedHeadroom); + const usize preferredBytes = std::min(desiredBytes, maximumTriangles * k_DihedralGatherBytesPerFace); + const uint64 requestedBytes = std::min(observedPlan.FaceCapacity == 0 ? preferredBytes : observedPlan.PayloadBytes, budget.maximumWorkingMemoryBytes()); + // A snapshot avoids known excess admission; a racing reservation can still reduce the actual grant. + auto reservation = budget.reserveWorkingMemory(requestedBytes); + const usize grantedBytes = static_cast(std::min(reservation.sizeBytes(), maximum)); + const auto plan = planDihedralWorkingMemory(triangles, verts, grantedBytes); + const bool cacheVertices = plan.CacheVertices; + const usize bufferTriangles = plan.FaceCapacity; + if(bufferTriangles == 0) + { + return MakeErrorResult( + -65825, + fmt::format("Triangle dihedral-angle calculation requested {} working bytes and received {}. At least {} bytes are needed for one gathered face, or {} bytes for cached vertices and one face.", + requestedBytes, grantedBytes, k_DihedralGatherBytesPerFace, vertexBytes + k_DihedralFastBytesPerFace)); + } + const usize pageVertices = plan.PageVertices; + // The checked plan fits the actual grant; release any remainder before allocations or I/O. + reservation.shrinkTo(plan.PayloadBytes); + Result<> result; + try + { + std::vector vertices(cacheVertices ? numVertices * 3 : 0); + if(cacheVertices) + { + result = MergeResults(std::move(result), verts.copyIntoBuffer(0, {vertices.data(), vertices.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65820, "Error reading vertex coordinates for triangle dihedral angle calculation.")); + } + } + std::vector order(cacheVertices ? 0 : bufferTriangles * 3); + std::vector gathered(cacheVertices ? 0 : bufferTriangles * 9); + std::vector page(pageVertices * 3); + std::vector triangleBuffer(bufferTriangles * 3); + std::vector angleBuffer(bufferTriangles); + for(usize start = 0; start < numTriangles;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(bufferTriangles, numTriangles - start); + result = MergeResults(std::move(result), triangles.copyIntoBuffer(start * 3, nonstd::span(triangleBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65821, fmt::format("Error reading connectivity for triangle dihedral angle calculation at triangle {} ({} triangles).", start, count))); + } + + for(usize triangleIndex = 0; triangleIndex < count; triangleIndex++) + { + for(usize vertex = 0; vertex < 3; ++vertex) + { + const auto vertexIndex = triangleBuffer[(triangleIndex * 3) + vertex]; + if(vertexIndex >= numVertices) + { + return MergeResults(std::move(result), + MakeErrorResult(-65823, fmt::format("Triangle {} references vertex index {}, but the vertex tuple count is {}. Valid vertex indices are less than {}.", + start + triangleIndex, vertexIndex, numVertices, numVertices))); + } + } + } + if(!cacheVertices) + { + result = MergeResults(std::move(result), MeshingUtilities::detail::GatherVertexCoordinates(verts, {triangleBuffer.data(), count * 3}, {order.data(), count * 3}, {gathered.data(), count * 9}, + {page.data(), page.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), + MakeErrorResult(-65820, fmt::format("Error reading vertex coordinates for triangle dihedral-angle calculation at triangle {} ({} triangles).", start, count))); + } + } + nx::core::Vec3 vecAB = {0.0f, 0.0f, 0.0f}; + nx::core::Vec3 vecAC = {0.0f, 0.0f, 0.0f}; + nx::core::Vec3 vecBC = {0.0f, 0.0f, 0.0f}; + for(usize triangleIndex = 0; triangleIndex < count; triangleIndex++) + { + std::array vertCoords; + for(usize vertex = 0; vertex < 3; ++vertex) + { + const usize offset = cacheVertices ? triangleBuffer[(triangleIndex * 3) + vertex] * 3 : (triangleIndex * 9) + (vertex * 3); + const auto& coordinates = cacheVertices ? vertices : gathered; + vertCoords.at(vertex) = Point3Df{coordinates[offset], coordinates[offset + 1], coordinates[offset + 2]}; + } + // Keep float coordinate subtraction and the exact in-core operation order. + for(usize i = 0; i < 3; i++) + { + vecAB[i] = vertCoords[0][i] - vertCoords[1][i]; + vecAC[i] = vertCoords[0][i] - vertCoords[2][i]; + vecBC[i] = vertCoords[1][i] - vertCoords[2][i]; + } + + const float64 magAB = vecAB.magnitude(); + const float64 magAC = vecAC.magnitude(); + const float64 magBC = vecBC.magnitude(); + + if(magAB == 0.0f || magAC == 0.0f || magBC == 0.0f) + { + angleBuffer[triangleIndex] = std::nan("0"); + } + else + { + std::array dihedralAnglesVec{}; + dihedralAnglesVec[0] = k_radToDeg * acos((std::fabs(vecAB.dot(vecAC)) / (magAB * magAC))); + + // 180 - angle because AB points out of vertex and BC points into vertex, so angle is actually angle outside of triangle + dihedralAnglesVec[1] = 180.0 - (k_radToDeg * acos((std::fabs(vecAB.dot(vecBC)) / (magAB * magBC)))); + dihedralAnglesVec[2] = k_radToDeg * acos((std::fabs(vecBC.dot(vecAC)) / (magBC * magAC))); + + angleBuffer[triangleIndex] = *std::ranges::min_element(dihedralAnglesVec); + } + } + result = MergeResults(std::move(result), dihedralAngles.copyFromBuffer(start, nonstd::span(angleBuffer.data(), count))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65822, fmt::format("Error writing dihedral angles at triangle {} ({} triangles).", start, count))); + } + start += count; + } + return result; + } catch(const std::bad_alloc&) + { + return MergeResults(std::move(result), MakeErrorResult(-65826, "Triangle dihedral-angle calculation could not allocate its admitted working buffers. Reduce the mesh size or free memory.")); + } catch(const std::length_error&) + { + return MergeResults(std::move(result), MakeErrorResult(-65826, "Triangle dihedral-angle calculation exceeded the supported buffer allocation size. Reduce the mesh size.")); + } +} } // namespace // ----------------------------------------------------------------------------- @@ -115,6 +331,14 @@ Result<> TriangleDihedralAngle::operator()() const DataPath dihedralAnglesArrayPath = pTriangleGeometryDataPath.createChildPath(faceAttributeMatrix->getName()).createChildPath(pMinDihedralAnglesName); auto& dihedralAnglesRef = m_DataStructure.getDataAs(dihedralAnglesArrayPath)->getDataStoreRef(); + const bool usesOutOfCoreStore = AnyOutOfCore({triangleGeom.getFaces(), triangleGeom.getVertices(), m_DataStructure.getDataAs(dihedralAnglesArrayPath)}); + const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + return CalculateDihedralAnglesInBlocks(triangleGeom, dihedralAnglesRef, m_ShouldCancel); + } + ParallelDataAlgorithm dataAlg; dataAlg.setRange(0ULL, static_cast(triangleGeom.getNumberOfFaces())); dataAlg.execute(CalculateDihedralAnglesImpl(&triangleGeom, dihedralAnglesRef, m_ShouldCancel)); diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleNormal.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleNormal.cpp index 01b97eff4b..8971e30f57 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleNormal.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/TriangleNormal.cpp @@ -3,7 +3,6 @@ #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" #include "simplnx/Utilities/Meshing/TriangleUtilities.hpp" -#include "simplnx/Utilities/ParallelDataAlgorithm.hpp" using namespace nx::core; @@ -31,10 +30,5 @@ Result<> TriangleNormal::operator()() DataPath pNormalsArrayPath = pTriangleGeometryDataPath.createChildPath(faceAttributeMatrix->getName()).createChildPath(pNormalsName); auto& normalsRef = m_DataStructure.getDataAs(pNormalsArrayPath)->getDataStoreRef(); - // Parallel algorithm to calculate normals - ParallelDataAlgorithm dataAlg; - dataAlg.setRange(0ULL, static_cast(triangleGeom.getNumberOfFaces())); - dataAlg.execute(MeshingUtilities::CalculateNormalsImpl(triangleGeom.getFaces()->getDataStoreRef(), triangleGeom.getVertices()->getDataStoreRef(), normalsRef, m_ShouldCancel)); - - return {}; + return MeshingUtilities::CalculateNormals(*triangleGeom.getFaces(), *triangleGeom.getVertices(), normalsRef, m_ShouldCancel); } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/VerifyTriangleWinding.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/VerifyTriangleWinding.cpp index 704a77007f..1b20fe775d 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/VerifyTriangleWinding.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/VerifyTriangleWinding.cpp @@ -1,21 +1,330 @@ #include "VerifyTriangleWinding.hpp" #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include #include "SimplnxCore/Filters/ReverseTriangleWindingFilter.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/DataStructure/IDataStore.hpp" +#include "simplnx/Filter/IFilter.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataArrayUtilities.hpp" #include "simplnx/Utilities/Meshing/TriangleUtilities.hpp" #include "simplnx/Utilities/Meshing/VertexUtilities.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" +#include "simplnx/Utilities/ThrottledMessageHandler.hpp" using namespace nx::core; namespace { +using MeshIndexType = IGeometry::MeshIndexType; + +// Keep the float32 operations and association identical to FindTriangleVolume. +float32 FindGatheredTriangleVolume(const float32* a, const float32* b, const float32* c) +{ + const std::array matrix = {b[0] - a[0], c[0] - a[0], 0.0f - a[0], b[1] - a[1], c[1] - a[1], 0.0f - a[1], b[2] - a[2], c[2] - a[2], 0.0f - a[2]}; + const float32 determinant = + (matrix[0] * (matrix[4] * matrix[8] - matrix[5] * matrix[7])) - (matrix[1] * (matrix[3] * matrix[8] - matrix[5] * matrix[6])) + (matrix[2] * (matrix[3] * matrix[7] - matrix[4] * matrix[6])); + return determinant / 6.0f; +} + +void AccumulateGatheredTriangleVolume(const float32* coordinates, const int32* ids, usize idComponents, std::vector& volumes) +{ + const float32* a = coordinates; + const float32* b = coordinates + 3; + const float32* c = coordinates + 6; + const int32 first = ids[0]; + if(idComponents == 1) + { + if(first >= 0) + { + volumes[first] += FindGatheredTriangleVolume(a, c, b); + } + return; + } + const int32 second = ids[1]; + const bool firstInRange = first >= 0 && static_cast(first) < volumes.size(); + const bool secondInRange = second >= 0 && static_cast(second) < volumes.size(); + if(first < 0 && secondInRange) + { + volumes[second] += FindGatheredTriangleVolume(a, c, b); + } + else if(second < 0 && firstInRange) + { + volumes[first] += FindGatheredTriangleVolume(a, b, c); + } + else if(firstInRange && secondInRange) + { + volumes[first] += FindGatheredTriangleVolume(a, b, c); + volumes[second] += FindGatheredTriangleVolume(a, c, b); + } +} + +bool MergeWindingTransfer(Result<>& result, Result<> transfer, const std::string& geometryPath, const char* phasePtr, usize firstTuple, usize tupleCount) +{ + if(transfer.invalid()) + { + transfer = MergeResults(std::move(transfer), MakeErrorResult(-56325, fmt::format("Triangle winding for geometry '{}': {} failed at tuple {} ({} tuples). Check the preceding storage diagnostic.", + geometryPath, phasePtr, firstTuple, tupleCount))); + } + result = MergeResults(std::move(result), std::move(transfer)); + return result.valid(); +} + +/** + * @brief Accumulates signed feature volumes and reverses selected faces through bounded bulk transfers. + * @param triangles Supplies three vertex indexes per face and receives complete reversed tuples. + * @param vertices Supplies three float32 coordinates per vertex. + * @param idsStore Supplies one region ID or two face labels per face. + * @param geometryPath Identifies the geometry in storage and allocation errors. + * @param shouldCancel Stops work before the next transfer or bounded computation check. + * @param messageHandler Receives phase announcements and throttled progress. + * @return Storage warnings, or an error for invalid shapes, indexes, transfers, or insufficient working memory. + * @note The working reservation bounds block buffers; the dense feature-volume array scales with the largest feature ID. + * Cancellation discards incomplete volume sums. Completed reversal blocks remain written. + */ +Result<> ReverseNegativeVolumesBounded(AbstractDataStore& triangles, const Float32AbstractDataStore& vertices, const Int32AbstractDataStore& idsStore, const std::string& geometryPath, + const std::atomic_bool& shouldCancel, const IFilter::MessageHandler& messageHandler) +{ + Result<> result; + const usize triangleCount = triangles.getNumberOfTuples(); + const usize vertexCount = vertices.getNumberOfTuples(); + const usize idComponents = idsStore.getNumberOfComponents(); + if(shouldCancel || triangleCount == 0) + { + return result; + } + const usize maxSize = std::numeric_limits::max(); + if(triangles.getNumberOfComponents() != 3 || vertices.getNumberOfComponents() != 3 || (idComponents != 1 && idComponents != 2) || idsStore.getNumberOfTuples() != triangleCount || + triangleCount > maxSize / 3 || vertexCount > maxSize / 3) + { + return MakeErrorResult(-56322, + fmt::format("Triangle winding for geometry '{}': expected XYZ vertices, three corners per face, and one or two IDs per face. Found {} vertices with {} components, " + "{} faces with {} components, and {} ID tuples with {} components. Correct the array shapes and ensure flat indexes fit in memory addressing.", + geometryPath, vertexCount, vertices.getNumberOfComponents(), triangleCount, triangles.getNumberOfComponents(), idsStore.getNumberOfTuples(), idComponents)); + } + + // Face values, corner permutation, gathered XYZ, at most 3B vertex tuples, and IDs. + // The existing dense feature accumulator is separate from this new bounded payload. + const usize bytesPerTriangle = (3 * sizeof(MeshIndexType)) + (3 * sizeof(usize)) + (18 * sizeof(float32)) + (idComponents * sizeof(int32)); + auto& budget = CacheMemoryBudgetManager::instance(); + const usize requestedCapacity = static_cast(std::min(std::min(triangleCount, maxSize / bytesPerTriangle), budget.maximumWorkingMemoryBytes() / bytesPerTriangle)); + const usize requestedBytes = requestedCapacity * bytesPerTriangle; + auto reservation = budget.reserveWorkingMemory(requestedBytes); + const usize blockCapacity = static_cast(std::min(triangleCount, reservation.sizeBytes() / bytesPerTriangle)); + if(blockCapacity == 0) + { + return MakeErrorResult(-56323, fmt::format("Triangle winding for geometry '{}': requested {} working bytes and received {}; at least {} bytes are required for one triangle. " + "Release other working reservations or increase the configured cache budget before retrying.", + geometryPath, requestedBytes, reservation.sizeBytes(), bytesPerTriangle)); + } + const usize pageCapacity = std::min(3 * blockCapacity, vertexCount); + const char* phasePtr = "allocating bounded buffers"; + try + { + auto faces = std::make_unique>(3 * blockCapacity); + auto cornerOrder = std::make_unique>(3 * blockCapacity); + auto gathered = std::make_unique>(9 * blockCapacity); + auto vertexPage = std::make_unique>(3 * pageCapacity); + auto ids = std::make_unique>(idComponents * blockCapacity); + ThrottledMessageHandler progressThrottle(messageHandler); + phasePtr = "scanning feature IDs"; + messageHandler.sendInfoMessage("Scanning feature IDs"); + progressThrottle.reset(triangleCount, "Scanning feature IDs"); + int32 maxFeature = 0; + for(usize begin = 0; begin < triangleCount;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(blockCapacity, triangleCount - begin); + if(!MergeWindingTransfer(result, idsStore.copyIntoBuffer(begin * idComponents, {ids->data(), count * idComponents}), geometryPath, phasePtr, begin, count)) + { + return result; + } + for(usize index = 0; index < count * idComponents; ++index) + { + maxFeature = std::max(maxFeature, (*ids)[index]); + } + begin += count; + progressThrottle.updatePercent(begin); + } + if(shouldCancel) + { + return result; + } + messageHandler.sendProgressPercent("Scanning feature IDs", triangleCount, triangleCount); + phasePtr = "allocating feature volumes"; + const usize featureCount = static_cast(maxFeature) + 1; + if(featureCount > maxSize / sizeof(float32)) + { + return MergeResults(std::move(result), + MakeErrorResult(-56324, fmt::format("Triangle winding for geometry '{}': {} feature volumes exceed the addressable allocation size. Renumber sparse feature IDs.", + geometryPath, featureCount))); + } + std::vector volumes(featureCount, 0.0f); + phasePtr = "calculating feature volumes"; + messageHandler.sendInfoMessage("Calculating feature volumes"); + progressThrottle.reset(triangleCount, "Calculating feature volumes"); + for(usize begin = 0; begin < triangleCount;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(blockCapacity, triangleCount - begin); + if(!MergeWindingTransfer(result, triangles.copyIntoBuffer(begin * 3, {faces->data(), count * 3}), geometryPath, "reading volume faces", begin, count) || + !MergeWindingTransfer(result, idsStore.copyIntoBuffer(begin * idComponents, {ids->data(), count * idComponents}), geometryPath, "reading volume IDs", begin, count)) + { + return result; + } + for(usize corner = 0; corner < 3 * count; ++corner) + { + if((*faces)[corner] >= vertexCount) + { + return MergeResults( + std::move(result), + MakeErrorResult(-56326, fmt::format("Triangle winding for geometry '{}': face {} references vertex {}, but only {} vertices exist. Repair the face connectivity before retrying.", + geometryPath, begin + (corner / 3), (*faces)[corner], vertexCount))); + } + } + if(shouldCancel) + { + return result; + } + std::iota(cornerOrder->data(), cornerOrder->data() + (3 * count), usize{0}); + // Only I/O requests are sorted; feature additions retain their original order. + // Cancellation waits for this bounded sort, then is checked before every page read. + std::sort(cornerOrder->data(), cornerOrder->data() + (3 * count), [&faces](usize left, usize right) { return (*faces)[left] < (*faces)[right]; }); + usize loadedPage = maxSize; + for(usize request = 0; request < 3 * count; ++request) + { + const usize corner = (*cornerOrder)[request]; + const usize vertex = static_cast((*faces)[corner]); + const usize pageBegin = (vertex / pageCapacity) * pageCapacity; + if(pageBegin != loadedPage) + { + if(shouldCancel) + { + return result; + } + const usize pageCount = std::min(pageCapacity, vertexCount - pageBegin); + if(!MergeWindingTransfer(result, vertices.copyIntoBuffer(pageBegin * 3, {vertexPage->data(), pageCount * 3}), geometryPath, "reading volume vertices", pageBegin, pageCount)) + { + return result; + } + loadedPage = pageBegin; + } + std::copy_n(vertexPage->data() + ((vertex - pageBegin) * 3), 3, gathered->data() + (corner * 3)); + } + for(usize triangle = 0; triangle < count; ++triangle) + { + if(triangle % 4096 == 0 && shouldCancel) + { + return result; + } + AccumulateGatheredTriangleVolume(gathered->data() + (triangle * 9), ids->data() + (triangle * idComponents), idComponents, volumes); + } + begin += count; + progressThrottle.updatePercent(begin); + } + if(shouldCancel) + { + return result; + } + messageHandler.sendProgressPercent("Calculating feature volumes", triangleCount, triangleCount); + if(shouldCancel) + { + return result; + } + if(std::none_of(volumes.begin() + 1, volumes.end(), [](float32 volume) { return volume < 0.0f; })) + { + messageHandler.sendInfoMessage("Triangle winding reversal complete"); + return result; + } + phasePtr = "reversing triangle winding"; + messageHandler.sendInfoMessage("Implementing reversal in bounded blocks"); + progressThrottle.reset(triangleCount, "Reversing triangle winding"); + for(usize begin = 0; begin < triangleCount;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(blockCapacity, triangleCount - begin); + if(!MergeWindingTransfer(result, triangles.copyIntoBuffer(begin * 3, {faces->data(), count * 3}), geometryPath, "reading reversal faces", begin, count) || + !MergeWindingTransfer(result, idsStore.copyIntoBuffer(begin * idComponents, {ids->data(), count * idComponents}), geometryPath, "reading reversal IDs", begin, count)) + { + return result; + } + bool changed = false; + for(usize triangle = 0; triangle < count; ++triangle) + { + if(triangle % 4096 == 0 && shouldCancel) + { + return result; + } + for(usize component = 0; component < idComponents; ++component) + { + const int32 id = (*ids)[(triangle * idComponents) + component]; + if(id > 0 && static_cast(id) < volumes.size() && volumes[id] < 0.0f) + { + std::swap((*faces)[triangle * 3], (*faces)[(triangle * 3) + 2]); + changed = true; + break; + } + } + } + if(shouldCancel) + { + return result; + } + if(changed && !MergeWindingTransfer(result, triangles.copyFromBuffer(begin * 3, {faces->data(), count * 3}), geometryPath, "writing reversed faces", begin, count)) + { + return result; + } + begin += count; + progressThrottle.updatePercent(begin); + } + messageHandler.sendProgressPercent("Reversing triangle winding", triangleCount, triangleCount); + messageHandler.sendInfoMessage("Triangle winding reversal complete"); + return result; + } catch(const std::bad_alloc&) + { + return MergeResults( + std::move(result), + MakeErrorResult(-56324, fmt::format("Triangle winding for geometry '{}': allocation failed while {} ({} reserved working bytes). Free memory or reduce sparse feature IDs before retrying.", + geometryPath, phasePtr, reservation.sizeBytes()))); + } catch(const std::length_error&) + { + return MergeResults( + std::move(result), + MakeErrorResult(-56324, fmt::format("Triangle winding for geometry '{}': allocation length is unsupported while {}. Reduce the geometry size or renumber sparse feature IDs before retrying.", + geometryPath, phasePtr))); + } +} + class ReverseWindingImpl { public: @@ -172,38 +481,55 @@ Result<> VerifyTriangleWinding::operator()() } m_MessageHandler.sendInfoMessage("Assessing reversal - calculating feature volumes"); - // Get max group (feature id != 0) - int32 maxFeature = 0; - for(int32 i = 0; i < idsStore.getSize(); i++) + const auto& vertices = triGeom.getVertices()->getDataStoreRef(); + const bool usesOutOfCoreStore = + triangles.getStoreType() == IDataStore::StoreType::OutOfCore || vertices.getStoreType() == IDataStore::StoreType::OutOfCore || idsStore.getStoreType() == IDataStore::StoreType::OutOfCore; + const bool useOutOfCoreAlgorithm = ShouldUseOutOfCoreAlgorithm(usesOutOfCoreStore); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) { - maxFeature = std::max(idsStore[i], maxFeature); + windingResult = + MergeResults(std::move(windingResult), ReverseNegativeVolumesBounded(triangles, vertices, idsStore, m_InputValues->TargetGeometryPath.toString(), m_ShouldCancel, m_MessageHandler)); + if(windingResult.invalid() || m_ShouldCancel) + { + return windingResult; + } } - - std::vector volumes(maxFeature + 1); - auto volumeResult = MeshingUtilities::CalculateFeatureVolumes(triangles, triGeom.getVertices()->getDataStoreRef(), idsStore, volumes, m_ShouldCancel); - if(volumeResult.invalid()) + else { - return volumeResult; - } + // Get max group (feature id != 0) + int32 maxFeature = 0; + for(int32 i = 0; i < idsStore.getSize(); i++) + { + maxFeature = std::max(idsStore[i], maxFeature); + } - m_MessageHandler.sendInfoMessage("Implementing reversal in parallel"); - std::vector reversalTargets = {}; - reversalTargets.reserve(volumes.size()); - for(usize i = 1; i < volumes.size(); i++) // zero is an invalid feature label - { - if(volumes[i] < 0.0f) + std::vector volumes(maxFeature + 1); + auto volumeResult = MeshingUtilities::CalculateFeatureVolumes(triangles, triGeom.getVertices()->getDataStoreRef(), idsStore, volumes, m_ShouldCancel); + if(volumeResult.invalid()) { - reversalTargets.push_back(i); + return volumeResult; } - } - reversalTargets.shrink_to_fit(); - if(!reversalTargets.empty()) - { - // Parallel algorithm to reverse based on feature label - ParallelDataAlgorithm dataAlg; - dataAlg.setRange(0ULL, static_cast(triangles.getNumberOfTuples())); - dataAlg.execute(::ReverseWindingImpl(triangles, idsStore, reversalTargets, m_ShouldCancel)); + m_MessageHandler.sendInfoMessage("Implementing reversal in parallel"); + std::vector reversalTargets = {}; + reversalTargets.reserve(volumes.size()); + for(usize i = 1; i < volumes.size(); i++) // zero is an invalid feature label + { + if(volumes[i] < 0.0f) + { + reversalTargets.push_back(i); + } + } + reversalTargets.shrink_to_fit(); + + if(!reversalTargets.empty()) + { + // Parallel algorithm to reverse based on feature label + ParallelDataAlgorithm dataAlg; + dataAlg.setRange(0ULL, triangles.getNumberOfTuples()); + dataAlg.execute(::ReverseWindingImpl(triangles, idsStore, reversalTargets, m_ShouldCancel)); + } } if(m_InputValues->RepairNormals) @@ -211,10 +537,7 @@ Result<> VerifyTriangleWinding::operator()() m_MessageHandler.sendInfoMessage("Recalculating normals"); auto& normals = m_DataStructure.getDataAs(m_InputValues->TriangleNormalsPath)->getDataStoreRef(); - // Parallel algorithm to calculate normals - ParallelDataAlgorithm dataAlg; - dataAlg.setRange(0ULL, static_cast(triGeom.getNumberOfFaces())); - dataAlg.execute(MeshingUtilities::CalculateNormalsImpl(triangles, triGeom.getVertices()->getDataStoreRef(), normals, m_ShouldCancel)); + windingResult = MergeResults(std::move(windingResult), MeshingUtilities::CalculateNormals(*triGeom.getFaces(), *triGeom.getVertices(), normals, m_ShouldCancel)); } return windingResult; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.cpp index ef877def1f..dbba1b71d4 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.cpp @@ -1,10 +1,17 @@ #include "WriteAvizoRectilinearCoordinate.hpp" #include "simplnx/Common/Bit.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" +#include +#include +#include +#include + using namespace nx::core; WriteAvizoRectilinearCoordinate::WriteAvizoRectilinearCoordinate(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, @@ -28,62 +35,130 @@ Result<> WriteAvizoRectilinearCoordinate::generateHeader(FILE* outputFile) const { if constexpr(endian::big == endian::native) { - fprintf(outputFile, "# AmiraMesh BINARY 2.1\n"); + if(auto outputResult = printOutput(outputFile, "# AmiraMesh BINARY 2.1\n"); outputResult.invalid()) + { + return outputResult; + } } else { - fprintf(outputFile, "# AmiraMesh BINARY-LITTLE-ENDIAN 2.1\n"); + if(auto outputResult = printOutput(outputFile, "# AmiraMesh BINARY-LITTLE-ENDIAN 2.1\n"); outputResult.invalid()) + { + return outputResult; + } } } else { - fprintf(outputFile, "# AmiraMesh 3D ASCII 2.0\n"); + if(auto outputResult = printOutput(outputFile, "# AmiraMesh 3D ASCII 2.0\n"); outputResult.invalid()) + { + return outputResult; + } + } + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, "# Dimensions in x-, y-, and z-direction\n"); outputResult.invalid()) + { + return outputResult; } - fprintf(outputFile, "\n"); - fprintf(outputFile, "# Dimensions in x-, y-, and z-direction\n"); SizeVec3 geoDim = geom.getDimensions(); - fprintf(outputFile, "define Lattice %llu %llu %llu\n", static_cast(geoDim[0]), static_cast(geoDim[1]), static_cast(geoDim[2])); - fprintf(outputFile, "define Coordinates %llu\n\n", static_cast(geoDim[0] + geoDim[1] + geoDim[2])); + if(auto outputResult = + printOutput(outputFile, "define Lattice %llu %llu %llu\n", static_cast(geoDim[0]), static_cast(geoDim[1]), static_cast(geoDim[2])); + outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, "define Coordinates %llu\n\n", static_cast(geoDim[0] + geoDim[1] + geoDim[2])); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, "Parameters {\n"); - fprintf(outputFile, " DREAM3DParams {\n"); - fprintf(outputFile, " Author \"DREAM3D-NX SimplnxCore Version 7.0.0\",\n"); + if(auto outputResult = printOutput(outputFile, "Parameters {\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " DREAM3DParams {\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " Author \"DREAM3D-NX SimplnxCore Version 7.0.0\",\n"); outputResult.invalid()) + { + return outputResult; + } const std::time_t currentTime = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); const std::string timeString = std::ctime(¤tTime); // ctime() includes a final newline that is not part of the quoted value. - fprintf(outputFile, " DateTime \"%s\"\n", timeString.substr(0, timeString.length() - 1).c_str()); - fprintf(outputFile, " FeatureIds Path \"%s\"\n", m_InputValues->FeatureIdsArrayPath.toString().c_str()); - fprintf(outputFile, " }\n"); + if(auto outputResult = printOutput(outputFile, " DateTime \"%s\"\n", timeString.substr(0, timeString.length() - 1).c_str()); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " FeatureIds Path \"%s\"\n", m_InputValues->FeatureIdsArrayPath.toString().c_str()); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " }\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, " Units {\n"); - fprintf(outputFile, " Coordinates \"%s\"\n", m_InputValues->Units.c_str()); - fprintf(outputFile, " }\n"); + if(auto outputResult = printOutput(outputFile, " Units {\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " Coordinates \"%s\"\n", m_InputValues->Units.c_str()); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " }\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, " CoordType \"rectilinear\"\n"); - fprintf(outputFile, "}\n\n"); + if(auto outputResult = printOutput(outputFile, " CoordType \"rectilinear\"\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, "}\n\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, "Lattice { int FeatureIds } = @1\n"); - fprintf(outputFile, "Coordinates { float xyz } = @2\n\n"); + if(auto outputResult = printOutput(outputFile, "Lattice { int FeatureIds } = @1\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, "Coordinates { float xyz } = @2\n\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, "# Data section follows\n"); + if(auto outputResult = printOutput(outputFile, "# Data section follows\n"); outputResult.invalid()) + { + return outputResult; + } return {}; } Result<> WriteAvizoRectilinearCoordinate::writeData(FILE* outputFile) const { + Result<> result; const auto& geom = m_DataStructure.getDataRefAs(m_InputValues->GeometryPath); SizeVec3 dims = geom.getDimensions(); FloatVec3 origin = geom.getOrigin(); FloatVec3 res = geom.getSpacing(); - fprintf(outputFile, "@1 # FeatureIds in z, y, x with X moving fastest, then Y, then Z\n"); + if(auto outputResult = printOutput(outputFile, "@1 # FeatureIds in z, y, x with X moving fastest, then Y, then Z\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } const auto& featureIds = m_DataStructure.getDataRefAs(m_InputValues->FeatureIdsArrayPath); const usize totalPoints = featureIds.getNumberOfTuples(); - // Source read failures stop output. The legacy stdio path does not report short writes. constexpr usize k_ChunkSize = 65536; std::vector buffer(k_ChunkSize); const auto& featureIdsStore = featureIds.getDataStoreRef(); @@ -94,15 +169,22 @@ Result<> WriteAvizoRectilinearCoordinate::writeData(FILE* outputFile) const { if(m_ShouldCancel) { - return {}; + return result; } const usize count = std::min(k_ChunkSize, totalPoints - offset); Result<> readResult = featureIdsStore.copyIntoBuffer(offset, nonstd::span(buffer.data(), count)); if(readResult.invalid()) { - return readResult; + return MergeResults(std::move(result), std::move(readResult)); + } + for(auto& warning : readResult.warnings()) + { + result.warnings().push_back(std::move(warning)); + } + if(auto outputResult = writeOutputChecked(outputFile, buffer.data(), sizeof(int32), count); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); } - fwrite(buffer.data(), sizeof(int32), count, outputFile); } } else @@ -113,45 +195,78 @@ Result<> WriteAvizoRectilinearCoordinate::writeData(FILE* outputFile) const { if(m_ShouldCancel) { - return {}; + return result; } const usize count = std::min(k_ChunkSize, totalPoints - offset); Result<> readResult = featureIdsStore.copyIntoBuffer(offset, nonstd::span(buffer.data(), count)); if(readResult.invalid()) { - return readResult; + return MergeResults(std::move(result), std::move(readResult)); + } + for(auto& warning : readResult.warnings()) + { + result.warnings().push_back(std::move(warning)); } for(usize i = 0; i < count; ++i) { - fprintf(outputFile, "%d", buffer[i]); + if(auto outputResult = printOutput(outputFile, "%d", buffer[i]); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } if(itemCount < 20) { - fprintf(outputFile, " "); + if(auto outputResult = printOutput(outputFile, " "); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } itemCount++; } else { - fprintf(outputFile, "\n"); + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } itemCount = 0; } } } } - fprintf(outputFile, "\n"); + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } - fprintf(outputFile, "@2 # x coordinates, then y, then z\n"); + if(auto outputResult = printOutput(outputFile, "@2 # x coordinates, then y, then z\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } if(m_InputValues->WriteBinaryFile) { + constexpr std::array axisNames = {'X', 'Y', 'Z'}; for(int d = 0; d < 3; ++d) { + if(dims[d] > std::numeric_limits::max() / sizeof(float32)) + { + auto overflowError = MakeErrorResult( + -5833, fmt::format("Cannot write {} coordinates to Avizo output '{}': {} float values exceed the byte-count limit.", axisNames[d], m_InputValues->OutputFile.string(), dims[d])); + return MergeResults(std::move(result), std::move(overflowError)); + } + const usize axisBytes = dims[d] * sizeof(float32); std::vector coords(dims[d]); for(usize i = 0; i < dims[d]; ++i) { coords[i] = origin[d] + (res[d] * i); } - fwrite(reinterpret_cast(coords.data()), sizeof(char), sizeof(char) * sizeof(float32) * dims[d], outputFile); - fprintf(outputFile, "\n"); + if(auto outputResult = writeOutputChecked(outputFile, reinterpret_cast(coords.data()), sizeof(char), axisBytes); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } } } else @@ -160,11 +275,17 @@ Result<> WriteAvizoRectilinearCoordinate::writeData(FILE* outputFile) const { for(usize i = 0; i < dims[d]; ++i) { - fprintf(outputFile, "%f ", origin[d] + (res[d] * i)); + if(auto outputResult = printOutput(outputFile, "%f ", origin[d] + (res[d] * i)); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } + } + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); } - fprintf(outputFile, "\n"); } } - return {}; + return result; } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp index c09a4a843f..0790699c4f 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp @@ -13,10 +13,10 @@ namespace nx::core * size is the current axis dimension. Binary output uses native endianness and * identifies it in the header. ASCII Feature IDs insert a newline after 21 values. * - * DataStore read results and C stdio return values are not inspected. A source - * or file-write failure can produce stale or partial output while returning - * success. Cancellation is checked between Feature ID chunks. It returns success - * and closes a file that does not include the remaining data or coordinates. + * The writer checks source reads and each output operation. A failure stops + * later reads and returns an error. Cancellation is checked between Feature ID + * windows. The filter wrapper rejects + * publication after cancellation. */ class SIMPLNXCORE_EXPORT WriteAvizoRectilinearCoordinate : public AvizoWriter { @@ -42,9 +42,7 @@ class SIMPLNXCORE_EXPORT WriteAvizoRectilinearCoordinate : public AvizoWriter /** * @brief Creates the output path and writes the Avizo file. - * @return Directory or file-open result from AvizoWriter. - * - * Cancellation and data-write failures are not distinguishable from success. + * @return Directory, source-read, output, or close status from AvizoWriter. */ Result<> operator()(); @@ -52,7 +50,7 @@ class SIMPLNXCORE_EXPORT WriteAvizoRectilinearCoordinate : public AvizoWriter /** * @brief Writes the rectilinear Avizo header. * @param outputFile Open binary-mode output stream. - * @return Success. C stdio failures are not inspected. + * @return Error if a header output operation fails. * @pre outputFile is not null. */ Result<> generateHeader(FILE* outputFile) const override; @@ -60,7 +58,7 @@ class SIMPLNXCORE_EXPORT WriteAvizoRectilinearCoordinate : public AvizoWriter /** * @brief Writes Feature IDs and X, Y, then Z coordinate arrays. * @param outputFile Open binary-mode output stream. - * @return Success after completion or cancellation. + * @return Source warnings and any read or output error. Cancellation ends the current output. * @pre outputFile is not null. */ Result<> writeData(FILE* outputFile) const override; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.cpp index a53528866a..7571a52c75 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.cpp @@ -1,9 +1,11 @@ #include "WriteAvizoUniformCoordinate.hpp" +#include "simplnx/Common/Result.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include +#include using namespace nx::core; @@ -28,64 +30,138 @@ Result<> WriteAvizoUniformCoordinate::generateHeader(FILE* outputFile) const { if constexpr(endian::big == endian::native) { - fprintf(outputFile, "# AmiraMesh BINARY 2.1\n"); + if(auto outputResult = printOutput(outputFile, "# AmiraMesh BINARY 2.1\n"); outputResult.invalid()) + { + return outputResult; + } } else { - fprintf(outputFile, "# AmiraMesh BINARY-LITTLE-ENDIAN 2.1\n"); + if(auto outputResult = printOutput(outputFile, "# AmiraMesh BINARY-LITTLE-ENDIAN 2.1\n"); outputResult.invalid()) + { + return outputResult; + } } } else { - fprintf(outputFile, "# AmiraMesh 3D ASCII 2.0\n"); + if(auto outputResult = printOutput(outputFile, "# AmiraMesh 3D ASCII 2.0\n"); outputResult.invalid()) + { + return outputResult; + } + } + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, "# Dimensions in x-, y-, and z-direction\n"); outputResult.invalid()) + { + return outputResult; } - fprintf(outputFile, "\n"); - fprintf(outputFile, "# Dimensions in x-, y-, and z-direction\n"); SizeVec3 dims = geom.getDimensions(); - fprintf(outputFile, "define Lattice %llu %llu %llu\n", static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])); + if(auto outputResult = + printOutput(outputFile, "define Lattice %llu %llu %llu\n", static_cast(dims[0]), static_cast(dims[1]), static_cast(dims[2])); + outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, "Parameters {\n"); - fprintf(outputFile, " DREAM3DParams {\n"); - fprintf(outputFile, " Author \"DREAM3D-NX SimplnxCore Version 7.0.0\",\n"); + if(auto outputResult = printOutput(outputFile, "Parameters {\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " DREAM3DParams {\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " Author \"DREAM3D-NX SimplnxCore Version 7.0.0\",\n"); outputResult.invalid()) + { + return outputResult; + } const std::time_t currentTime = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); const std::string timeString = std::ctime(¤tTime); // ctime() includes a final newline that is not part of the quoted value. - fprintf(outputFile, " DateTime \"%s\"\n", timeString.substr(0, timeString.length() - 1).c_str()); - fprintf(outputFile, " FeatureIds Path \"%s\"\n", m_InputValues->FeatureIdsArrayPath.toString().c_str()); - fprintf(outputFile, " }\n"); + if(auto outputResult = printOutput(outputFile, " DateTime \"%s\"\n", timeString.substr(0, timeString.length() - 1).c_str()); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " FeatureIds Path \"%s\"\n", m_InputValues->FeatureIdsArrayPath.toString().c_str()); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " }\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, " Units {\n"); - fprintf(outputFile, " Coordinates \"%s\"\n", m_InputValues->Units.c_str()); - fprintf(outputFile, " }\n"); + if(auto outputResult = printOutput(outputFile, " Units {\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " Coordinates \"%s\"\n", m_InputValues->Units.c_str()); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " }\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, " Content \"%llux%llux%llu int, uniform coordinates\",\n", static_cast(dims[0]), static_cast(dims[1]), - static_cast(dims[2])); + if(auto outputResult = printOutput(outputFile, " Content \"%llux%llux%llu int, uniform coordinates\",\n", static_cast(dims[0]), + static_cast(dims[1]), static_cast(dims[2])); + outputResult.invalid()) + { + return outputResult; + } FloatVec3 origin = geom.getOrigin(); FloatVec3 res = geom.getSpacing(); - fprintf(outputFile, " # Bounding Box is xmin xmax ymin ymax zmin zmax\n"); - fprintf(outputFile, " BoundingBox %f %f %f %f %f %f\n", origin[0], origin[0] + (res[0] * dims[0]), origin[1], origin[1] + (res[1] * dims[1]), origin[2], origin[2] + (res[2] * dims[2])); + if(auto outputResult = printOutput(outputFile, " # Bounding Box is xmin xmax ymin ymax zmin zmax\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, " BoundingBox %f %f %f %f %f %f\n", origin[0], origin[0] + (res[0] * dims[0]), origin[1], origin[1] + (res[1] * dims[1]), origin[2], + origin[2] + (res[2] * dims[2])); + outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, " CoordType \"uniform\"\n"); - fprintf(outputFile, "}\n\n"); + if(auto outputResult = printOutput(outputFile, " CoordType \"uniform\"\n"); outputResult.invalid()) + { + return outputResult; + } + if(auto outputResult = printOutput(outputFile, "}\n\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, "Lattice { int FeatureIds } = @1\n"); + if(auto outputResult = printOutput(outputFile, "Lattice { int FeatureIds } = @1\n"); outputResult.invalid()) + { + return outputResult; + } - fprintf(outputFile, "# Data section follows\n"); + if(auto outputResult = printOutput(outputFile, "# Data section follows\n"); outputResult.invalid()) + { + return outputResult; + } return {}; } Result<> WriteAvizoUniformCoordinate::writeData(FILE* outputFile) const { - fprintf(outputFile, "@1\n"); + Result<> result; + if(auto outputResult = printOutput(outputFile, "@1\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } const auto& featureIds = m_DataStructure.getDataRefAs(m_InputValues->FeatureIdsArrayPath); const usize totalPoints = featureIds.getNumberOfTuples(); - // Source read failures stop output. The legacy stdio path does not report short writes. constexpr usize k_ChunkSize = 65536; std::vector buffer(k_ChunkSize); const auto& featureIdsStore = featureIds.getDataStoreRef(); @@ -95,15 +171,22 @@ Result<> WriteAvizoUniformCoordinate::writeData(FILE* outputFile) const { if(m_ShouldCancel) { - return {}; + return result; } const usize count = std::min(k_ChunkSize, totalPoints - offset); Result<> readResult = featureIdsStore.copyIntoBuffer(offset, nonstd::span(buffer.data(), count)); if(readResult.invalid()) { - return readResult; + return MergeResults(std::move(result), std::move(readResult)); + } + for(auto& warning : readResult.warnings()) + { + result.warnings().push_back(std::move(warning)); + } + if(auto outputResult = writeOutputChecked(outputFile, buffer.data(), sizeof(int32), count); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); } - fwrite(buffer.data(), sizeof(int32), count, outputFile); } } else @@ -114,30 +197,46 @@ Result<> WriteAvizoUniformCoordinate::writeData(FILE* outputFile) const { if(m_ShouldCancel) { - return {}; + return result; } const usize count = std::min(k_ChunkSize, totalPoints - offset); Result<> readResult = featureIdsStore.copyIntoBuffer(offset, nonstd::span(buffer.data(), count)); if(readResult.invalid()) { - return readResult; + return MergeResults(std::move(result), std::move(readResult)); + } + for(auto& warning : readResult.warnings()) + { + result.warnings().push_back(std::move(warning)); } for(usize i = 0; i < count; ++i) { - fprintf(outputFile, "%d", buffer[i]); + if(auto outputResult = printOutput(outputFile, "%d", buffer[i]); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } if(itemCount < 20) { - fprintf(outputFile, " "); + if(auto outputResult = printOutput(outputFile, " "); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } itemCount++; } else { - fprintf(outputFile, "\n"); + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } itemCount = 0; } } } } - fprintf(outputFile, "\n"); - return {}; + if(auto outputResult = printOutput(outputFile, "\n"); outputResult.invalid()) + { + return MergeResults(std::move(result), std::move(outputResult)); + } + return result; } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.hpp index 8b296ba3a8..50d8963555 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.hpp @@ -14,10 +14,10 @@ namespace nx::core * endianness and identifies it in the header. ASCII output inserts a newline * after 21 Feature IDs. * - * DataStore read results and C stdio return values are not inspected. A source - * or file-write failure can produce stale or partial output while returning - * success. Cancellation is checked between Feature ID chunks and closes the - * partial file with a success result. + * The writer checks source reads and each output operation. A failure stops + * later reads and returns an error. Cancellation is checked between Feature ID + * windows. The filter wrapper rejects + * publication after cancellation. */ class SIMPLNXCORE_EXPORT WriteAvizoUniformCoordinate : public AvizoWriter { @@ -43,9 +43,7 @@ class SIMPLNXCORE_EXPORT WriteAvizoUniformCoordinate : public AvizoWriter /** * @brief Creates the output path and writes the Avizo file. - * @return Directory or file-open result from AvizoWriter. - * - * Cancellation and data-write failures are not distinguishable from success. + * @return Directory, source-read, output, or close status from AvizoWriter. */ Result<> operator()(); @@ -53,7 +51,7 @@ class SIMPLNXCORE_EXPORT WriteAvizoUniformCoordinate : public AvizoWriter /** * @brief Writes the uniform-coordinate Avizo header. * @param outputFile Open binary-mode output stream. - * @return Success. C stdio failures are not inspected. + * @return Error if a header output operation fails. * @pre outputFile is not null. */ Result<> generateHeader(FILE* outputFile) const override; @@ -61,7 +59,7 @@ class SIMPLNXCORE_EXPORT WriteAvizoUniformCoordinate : public AvizoWriter /** * @brief Writes Feature IDs in binary or ASCII form. * @param outputFile Open binary-mode output stream. - * @return Success after completion or cancellation. + * @return Source warnings and any read or output error. Cancellation ends the current output. * @pre outputFile is not null. */ Result<> writeData(FILE* outputFile) const override; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeKMeansFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeKMeansFilter.cpp index 9ddf6465b9..369d7fba92 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeKMeansFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ComputeKMeansFilter.cpp @@ -20,6 +20,9 @@ #include "simplnx/Utilities/SIMPLConversion.hpp" +#include +#include + #include using namespace nx::core; @@ -68,7 +71,7 @@ Parameters ComputeKMeansFilter::parameters() const params.insertSeparator(Parameters::Separator{"Input Parameter(s)"}); - params.insert(std::make_unique(k_InitClusters_Key, "Number of Clusters", "This will be the tuple size for Cluster Attribute Matrix and the values within", 0)); + params.insert(std::make_unique(k_InitClusters_Key, "Number of Clusters", "Number of clusters to create. The value must be greater than zero. The default is 1.", uint64{1})); params.insert(std::make_unique( k_DistanceMetric_Key, "Distance Metric", "Distance Metric type to be used for calculations", to_underlying(ClusterUtilities::DistanceMetric::Euclidean), ChoicesParameter::Choices{"Euclidean", "Squared Euclidean", "Manhattan", "Cosine", "Pearson", "Squared Pearson"})); // Choice order matches ClusterUtilities::DistanceMetric values. @@ -117,6 +120,11 @@ IFilter::PreflightResult ComputeKMeansFilter::preflightImpl(const DataStructure& auto pMeansArrayNameValue = filterArgs.value(k_MeansArrayName_Key); auto pSeedArrayNameValue = filterArgs.value(k_SeedArrayName_Key); + if(pInitClustersValue == 0) + { + return MakePreflightErrorResult(-54061, fmt::format("Number of Clusters ({}) for array '{}' must be greater than zero.", pInitClustersValue, pSelectedArrayPathValue.toString())); + } + PreflightResult preflightResult; nx::core::Result resultOutputActions; std::vector preflightUpdatedValues; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadDREAM3DFilter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadDREAM3DFilter.cpp index 36e5d24c19..8f8de32e7d 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadDREAM3DFilter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/Filters/ReadDREAM3DFilter.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace { @@ -82,12 +83,8 @@ IFilter::PreflightResult ReadDREAM3DFilter::preflightImpl(const DataStructure& d return {MakeErrorResult(k_NoImportPathError, "Import file path not provided.")}; } - // Preflight metadata is served from Dream3dPreflightCache instead of being - // read from the file on every pass: pipelines re-preflight on every - // parameter edit, and a full HDF5 metadata traversal per edit freezes the - // UI for seconds on high-latency storage (network mounts). After the first - // import, each preflight costs a single stat() (see Dream3dPreflightCache). - Result dataStructureResult = DREAM3D::Dream3dPreflightCache::Instance().fetch(importData.FilePath); + // Neutral metadata avoids source-policy decisions before the destination action can plan selected arrays. + auto dataStructureResult = DREAM3D::Dream3dPreflightCache::Instance().fetchNeutralMetadata(importData.FilePath); Result result; OutputActions& actions = result.value(); @@ -96,7 +93,8 @@ IFilter::PreflightResult ReadDREAM3DFilter::preflightImpl(const DataStructure& d { return {ConvertResultTo(ConvertResult(std::move(dataStructureResult)), {})}; } - auto importedDataStructure = dataStructureResult.value(); + result.warnings() = std::move(dataStructureResult.warnings()); + const auto& importedDataStructure = dataStructureResult.value().dataStructure; if(importData.ImportPolicy == Dream3dImportParameter::PathImportPolicy::IncludeList) { @@ -149,7 +147,9 @@ IFilter::PreflightResult ReadDREAM3DFilter::preflightImpl(const DataStructure& d } else { - return {MakeErrorResult(k_UnsupportedPathImportPolicyError, "The chosen PathImportPolicy is not supported by this filter. Please contact the developers.")}; + auto failure = MakeErrorResult(k_UnsupportedPathImportPolicyError, "The chosen PathImportPolicy is not supported by this filter. Please contact the developers."); + failure.warnings() = std::move(result.warnings()); + return {std::move(failure)}; } return {result}; diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.cpp b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.cpp index ec4dd6323a..4cb99dcee7 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.cpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.cpp @@ -1,10 +1,36 @@ #include "AvizoWriter.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + using namespace nx::core; +namespace +{ +std::string SystemErrorText(int errorNumber) +{ + if(errorNumber == 0) + { + return "No system error was reported"; + } + return fmt::format("System error {}: {}", errorNumber, std::error_code(errorNumber, std::generic_category()).message()); +} +} // namespace + // ----------------------------------------------------------------------------- AvizoWriter::AvizoWriter(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, AvizoWriterInputValues* inputValues) : m_DataStructure(dataStructure) @@ -23,36 +49,178 @@ const std::atomic_bool& AvizoWriter::getCancel() return m_ShouldCancel; } +void AvizoWriter::setOutputOperations(const OutputOperations* operations) noexcept +{ + m_OutputOperations = operations; +} + +usize AvizoWriter::writeOutput(FILE* stream, const void* data, usize elementSize, usize count) const +{ + if(m_OutputOperations != nullptr && m_OutputOperations->write != nullptr) + { + return m_OutputOperations->write(m_OutputOperations->context, stream, data, elementSize, count); + } + return std::fwrite(data, elementSize, count, stream); +} + +int AvizoWriter::printOutputV(FILE* stream, const char* format, std::va_list args) const +{ + if(m_OutputOperations != nullptr && m_OutputOperations->print != nullptr) + { + return m_OutputOperations->print(m_OutputOperations->context, stream, format, args); + } + return std::vfprintf(stream, format, args); +} + +int AvizoWriter::flushOutput(FILE* stream) const +{ + if(m_OutputOperations != nullptr && m_OutputOperations->flush != nullptr) + { + return m_OutputOperations->flush(m_OutputOperations->context, stream); + } + return std::fflush(stream); +} + +int AvizoWriter::closeOutput(FILE* stream) const +{ + if(m_OutputOperations != nullptr && m_OutputOperations->close != nullptr) + { + return m_OutputOperations->close(m_OutputOperations->context, stream); + } + return std::fclose(stream); +} + +Result<> AvizoWriter::printOutput(FILE* stream, const char* format, ...) const +{ + std::va_list argsPtr; + va_start(argsPtr, format); + const auto endArgs = MakeScopeGuard([&argsPtr]() noexcept { va_end(argsPtr); }); + errno = 0; + const int printed = printOutputV(stream, format, argsPtr); + const int errorNumber = errno; + if(printed < 0) + { + return MakeErrorResult(-5831, fmt::format("Cannot write text to Avizo output '{}'. {}.", m_InputValues->OutputFile.string(), SystemErrorText(errorNumber))); + } + return {}; +} + +Result<> AvizoWriter::writeOutputChecked(FILE* stream, const void* data, usize elementSize, usize count) const +{ + if(elementSize != 0 && count > std::numeric_limits::max() / elementSize) + { + return MakeErrorResult( + -5833, fmt::format("Cannot write {} elements of {} bytes each to Avizo output '{}': the byte count exceeds the platform limit.", count, elementSize, m_InputValues->OutputFile.string())); + } + + const usize expectedBytes = elementSize * count; + errno = 0; + const usize written = writeOutput(stream, data, elementSize, count); + const int errorNumber = errno; + if(written != count) + { + const std::string actualBytes = elementSize != 0 && written > std::numeric_limits::max() / elementSize ? "unrepresentable" : std::to_string(written * elementSize); + return MakeErrorResult(-5832, fmt::format("Cannot write Avizo output '{}': requested {} elements ({} bytes), wrote {} elements ({} bytes), with {} bytes per element. {}.", + m_InputValues->OutputFile.string(), count, expectedBytes, written, actualBytes, elementSize, SystemErrorText(errorNumber))); + } + return {}; +} + // ----------------------------------------------------------------------------- Result<> AvizoWriter::execute() { // Make sure any directory path is also available as the user may have just typed // in a path without actually creating the full path - Result<> createDirectoriesResult = CreateOutputDirectories(m_InputValues->OutputFile.parent_path()); - if(createDirectoriesResult.invalid()) + Result<> result = CreateOutputDirectories(m_InputValues->OutputFile.parent_path()); + if(result.invalid()) { - return createDirectoriesResult; + return result; } - FILE* outputFile = fopen(m_InputValues->OutputFile.string().c_str(), "wb"); - if(outputFile == nullptr) + const std::string outputPath = m_InputValues->OutputFile.string(); + errno = 0; + FILE* outputFilePtr = std::fopen(outputPath.c_str(), "wb"); + const int openErrorNumber = errno; + if(outputFilePtr == nullptr) { - return MakeErrorResult(-5830, fmt::format("Error creating output file '{}'. Check that the directory exists and you have write permissions.", m_InputValues->OutputFile.string())); + auto openError = MakeErrorResult(-5830, fmt::format("Cannot open Avizo output '{}'. Check the directory and write permissions. {}.", outputPath, SystemErrorText(openErrorNumber))); + return MergeResults(std::move(result), std::move(openError)); } - Result<> headerResult = generateHeader(outputFile); - if(headerResult.invalid()) + const auto closeOnExit = MakeScopeGuard([this, &outputFilePtr]() noexcept { + if(outputFilePtr != nullptr) + { + // A close attempt consumes the stream even if the operation reports failure. + FILE* ownedFilePtr = std::exchange(outputFilePtr, nullptr); + try + { + (void)closeOutput(ownedFilePtr); + } catch(...) + { + } + } + }); + + const char* stagePtr = "header"; + try + { + auto headerResult = generateHeader(outputFilePtr); + result = MergeResults(std::move(result), std::move(headerResult)); + if(result.valid()) + { + stagePtr = "data"; + auto dataResult = writeData(outputFilePtr); + result = MergeResults(std::move(result), std::move(dataResult)); + } + if(result.valid()) + { + stagePtr = "flush"; + errno = 0; + const int flushStatus = flushOutput(outputFilePtr); + const int flushErrorNumber = errno; + if(flushStatus != 0) + { + auto flushError = MakeErrorResult(-5834, fmt::format("Cannot flush Avizo output '{}'. {}.", outputPath, SystemErrorText(flushErrorNumber))); + result = MergeResults(std::move(result), std::move(flushError)); + } + } + } catch(const std::bad_alloc&) + { + throw; + } catch(const std::exception& exception) + { + auto stageError = MakeErrorResult(-5836, fmt::format("Cannot write Avizo output '{}' during {}: {}.", outputPath, stagePtr, exception.what())); + result = MergeResults(std::move(result), std::move(stageError)); + } catch(...) { - return headerResult; + auto stageError = MakeErrorResult(-5837, fmt::format("Cannot write Avizo output '{}' during {}: an unknown error occurred.", outputPath, stagePtr)); + result = MergeResults(std::move(result), std::move(stageError)); } - Result<> result = writeData(outputFile); - if(result.invalid()) + // The owner releases FILE before close because fclose can fail after consuming it. + FILE* closingFilePtr = std::exchange(outputFilePtr, nullptr); + try { - return result; + errno = 0; + const int closeStatus = closeOutput(closingFilePtr); + const int closeErrorNumber = errno; + if(closeStatus != 0) + { + auto closeError = MakeErrorResult(-5835, fmt::format("Cannot close Avizo output '{}'. {}.", outputPath, SystemErrorText(closeErrorNumber))); + result = MergeResults(std::move(result), std::move(closeError)); + } + } catch(const std::bad_alloc&) + { + throw; + } catch(const std::exception& exception) + { + auto closeError = MakeErrorResult(-5835, fmt::format("Cannot close Avizo output '{}': {}.", outputPath, exception.what())); + result = MergeResults(std::move(result), std::move(closeError)); + } catch(...) + { + auto closeError = MakeErrorResult(-5835, fmt::format("Cannot close Avizo output '{}': an unknown error occurred.", outputPath)); + result = MergeResults(std::move(result), std::move(closeError)); } - fclose(outputFile); - - return {}; + return result; } diff --git a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.hpp b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.hpp index 055bfb09e8..f676d48c6e 100644 --- a/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.hpp +++ b/src/Plugins/SimplnxCore/src/SimplnxCore/utils/AvizoWriter.hpp @@ -8,6 +8,9 @@ #include "simplnx/Parameters/FileSystemPathParameter.hpp" #include "simplnx/Parameters/StringParameter.hpp" +#include +#include + namespace nx::core { @@ -22,13 +25,26 @@ struct SIMPLNXCORE_EXPORT AvizoWriterInputValues /** * @class AvizoWriter - * @brief This filter writes out a native Avizo Uniform Coordinate data file. + * @brief Writes an Avizo data file with checked output and stream closure. */ - class SIMPLNXCORE_EXPORT AvizoWriter { public: + /** + * @brief Stores the data and options used by the writer. + * @param dataStructure Contains the geometry and Feature IDs. + * @param mesgHandler Receives filter messages. + * @param shouldCancel Signals cancellation between Feature ID windows. + * @param inputValues Specifies the output path, encoding, and input paths. + * @pre All referenced arguments outlive this writer. + */ AvizoWriter(DataStructure& dataStructure, const IFilter::MessageHandler& mesgHandler, const std::atomic_bool& shouldCancel, AvizoWriterInputValues* inputValues); + + /** + * @brief Destroys the writer without closing an active stream. + * + * execute() owns and closes its stream before this destructor runs. + */ virtual ~AvizoWriter() noexcept; AvizoWriter(const AvizoWriter&) = delete; @@ -36,18 +52,110 @@ class SIMPLNXCORE_EXPORT AvizoWriter AvizoWriter& operator=(const AvizoWriter&) = delete; AvizoWriter& operator=(AvizoWriter&&) noexcept = delete; + /** + * @brief Writes the file and closes its stream before returning. + * @return Directory, output, or close errors with earlier warnings retained. + * @throws std::bad_alloc if a stage or callback cannot allocate memory. + */ Result<> execute(); const std::atomic_bool& getCancel(); protected: + /** + * @struct OutputOperations + * @brief Supplies optional output callbacks for one writer. + * + * The writer does not own this bundle. Each null operation uses C stdio. + * A close callback consumes its FILE even when it reports an error or throws. + */ + struct OutputOperations + { + void* context = nullptr; + usize (*write)(void*, FILE*, const void*, usize, usize) = nullptr; + int (*print)(void*, FILE*, const char*, std::va_list) = nullptr; + int (*flush)(void*, FILE*) = nullptr; + int (*close)(void*, FILE*) = nullptr; + }; + + /** + * @brief Sets the callback bundle used by this writer. + * @param operations Bundle that remains valid through execute(), or null for C stdio. + */ + void setOutputOperations(const OutputOperations* operations) noexcept; + + /** + * @brief Writes elements through the selected operation. + * @param stream Open output stream. + * @param data Source elements. + * @param elementSize Size of one element in bytes. + * @param count Number of elements to write. + * @return Number of elements reported as written. + */ + usize writeOutput(FILE* stream, const void* data, usize elementSize, usize count) const; + + /** + * @brief Prints one formatted record through the selected operation. + * @param stream Open output stream. + * @param format C format string. + * @param args Arguments used once during this call. + * @return Number of bytes printed, or a negative value on failure. + */ + int printOutputV(FILE* stream, const char* format, std::va_list args) const; + + /** + * @brief Flushes the stream through the selected operation. + * @param stream Open output stream. + * @return Zero on success; nonzero on failure. + */ + int flushOutput(FILE* stream) const; + + /** + * @brief Consumes the stream through the selected close operation. + * @param stream Open output stream that the caller releases before this call. + * @return Zero on success; nonzero on failure. + */ + int closeOutput(FILE* stream) const; + + /** + * @brief Prints one formatted record and checks its result. + * @param stream Open output stream. + * @param format C format string. + * @return Error with the output path if printing fails. + */ + Result<> printOutput(FILE* stream, const char* format, ...) const; + + /** + * @brief Writes the exact element count after checking the byte count. + * @param stream Open output stream. + * @param data Source elements. + * @param elementSize Size of one element in bytes. + * @param count Number of elements to write. + * @return Error with the output path and counts if the transfer is incomplete. + */ + Result<> writeOutputChecked(FILE* stream, const void* data, usize elementSize, usize count) const; + + /** + * @brief Writes the selected Avizo header. + * @param outputFile Open stream owned by execute(). + * @return Header warnings or an output error. + */ virtual Result<> generateHeader(FILE* outputFile) const = 0; + + /** + * @brief Writes Feature IDs and any coordinate arrays. + * @param outputFile Open stream owned by execute(). + * @return Source warnings, source errors, or output errors. + */ virtual Result<> writeData(FILE* outputFile) const = 0; DataStructure& m_DataStructure; const AvizoWriterInputValues* m_InputValues = nullptr; const std::atomic_bool& m_ShouldCancel; const IFilter::MessageHandler& m_MessageHandler; + +private: + const OutputOperations* m_OutputOperations = nullptr; }; } // namespace nx::core diff --git a/src/Plugins/SimplnxCore/test/AvizoWriterFailureTestUtilities.hpp b/src/Plugins/SimplnxCore/test/AvizoWriterFailureTestUtilities.hpp new file mode 100644 index 0000000000..5a06c0ccc0 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/AvizoWriterFailureTestUtilities.hpp @@ -0,0 +1,190 @@ +#pragma once + +#include "SimplnxCore/SimplnxCore_test_dirs.hpp" + +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/Geometry/ImageGeom.hpp" +#include "simplnx/Parameters/FileSystemPathParameter.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @namespace nx::core::UnitTest::AvizoWriterFailure + * @brief Provides source-read failure fixtures for the Avizo filters. + */ +namespace nx::core::UnitTest::AvizoWriterFailure +{ +/** + * @class SecondBlockFailureStore + * @brief Reports an error after one complete Avizo source window. + * @tparam BaseStore Supplies resident or real HDF5 storage. + */ +template +class SecondBlockFailureStore : public BaseStore +{ +public: + using BaseStore::BaseStore; + + SecondBlockFailureStore() = default; + ~SecondBlockFailureStore() override = default; + SecondBlockFailureStore(const SecondBlockFailureStore&) = default; + SecondBlockFailureStore(SecondBlockFailureStore&&) = default; + SecondBlockFailureStore& operator=(const SecondBlockFailureStore&) = default; + SecondBlockFailureStore& operator=(SecondBlockFailureStore&&) = default; + + /** + * @brief Copies the first window and rejects the second read. + * @param startIndex Zero-based source element offset. + * @param buffer Receives the first window's feature IDs. + * @return Named error for the second read; otherwise the base store's Result. + */ + Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + if(readCalls < readOffsets.size()) + { + readOffsets[readCalls] = startIndex; + readCounts[readCalls] = buffer.size(); + } + ++readCalls; + if(readCalls == 2) + { + if(throwAllocation) + { + throw std::bad_alloc{}; + } + return MakeErrorResult(-92561, "Injected Avizo second-block read failure"); + } + Result<> result = BaseStore::copyIntoBuffer(startIndex, buffer); + if(result.valid()) + { + ++successfulReads; + firstBufferHasExpectedValues = std::all_of(buffer.begin(), buffer.end(), [](int32 value) { return value == 7; }); + } + return result; + } + + bool throwAllocation = false; + mutable bool firstBufferHasExpectedValues = false; + mutable usize readCalls = 0; + mutable usize successfulReads = 0; + mutable std::array readOffsets{}; + mutable std::array readCounts{}; +}; + +/** + * @brief Checks the real filter's publication and cleanup after a later source-read error. + * @tparam FilterType Selects the uniform or rectilinear Avizo filter. + * @tparam StoreType Supplies the observed source store. + * @param writeBinary Selects native binary feature IDs instead of ASCII. + * @param existingDestination Selects an existing file with sentinel bytes. + * @param store Supplies exactly two source windows. + * @param throwAllocation Selects the filter allocation-error path on the second read. + */ +template >> +void CheckSecondBlockFailure(bool writeBinary, bool existingDestination, std::shared_ptr store = std::make_shared(ShapeType{1, 1, 65537}, ShapeType{1}, std::optional{7}), + bool throwAllocation = false) +{ + namespace fs = std::filesystem; + const auto outputBase = fs::path(unit_test::k_BinaryTestOutputDir.view()); + fs::create_directories(outputBase); + const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); + const fs::path outputRoot = outputBase / ("AvizoReadFailure-" + std::to_string(stamp)); + REQUIRE(fs::create_directory(outputRoot)); + const auto cleanup = MakeScopeGuard([&]() noexcept { + try + { + std::error_code ignored; + fs::remove_all(outputRoot, ignored); + } catch(...) + { + } + }); + const fs::path destination = outputRoot / "output.am"; + const std::string sentinel = "preserved destination"; + if(existingDestination) + { + std::ofstream output(destination, std::ios::binary); + REQUIRE(output.is_open()); + output.write(sentinel.data(), static_cast(sentinel.size())); + output.close(); + REQUIRE(output.good()); + } + + DataStructure dataStructure; + auto* imageGeom = ImageGeom::Create(dataStructure, "ImageGeom"); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({65537, 1, 1}); + imageGeom->setOrigin({0.0F, 0.0F, 0.0F}); + imageGeom->setSpacing({1.0F, 1.0F, 1.0F}); + auto* cellData = AttributeMatrix::Create(dataStructure, "CellData", {1, 1, 65537}, imageGeom->getId()); + REQUIRE(cellData != nullptr); + imageGeom->setCellData(*cellData); + store->throwAllocation = throwAllocation; + auto* featureIds = Int32Array::Create(dataStructure, "FeatureIds", store, cellData->getId()); + REQUIRE(featureIds != nullptr); + + FilterType filter; + Arguments args = filter.getDefaultArguments(); + args.insertOrAssign(FilterType::k_OutputFile_Key, std::make_any(destination)); + args.insertOrAssign(FilterType::k_WriteBinaryFile_Key, std::make_any(writeBinary)); + args.insertOrAssign(FilterType::k_GeometryPath_Key, std::make_any(DataPath({"ImageGeom"}))); + args.insertOrAssign(FilterType::k_FeatureIdsArrayPath_Key, std::make_any(DataPath({"ImageGeom", "CellData", "FeatureIds"}))); + + const auto executeResult = filter.execute(dataStructure, args); + CHECK(store->readCalls == 2); + CHECK(store->successfulReads == 1); + CHECK(store->firstBufferHasExpectedValues); + CHECK(store->readOffsets == std::array{0, 65536}); + CHECK(store->readCounts == std::array{65536, 1}); + REQUIRE(executeResult.result.invalid()); + bool foundReadError = false; + for(const auto& error : executeResult.result.errors()) + { + foundReadError = foundReadError || (throwAllocation ? error.code == -272 : (error.code == -92561 && error.message.find("Injected Avizo second-block read failure") != std::string::npos)); + } + CHECK(foundReadError); + + if(existingDestination) + { + std::ifstream input(destination, std::ios::binary); + REQUIRE(input.is_open()); + const std::string actual((std::istreambuf_iterator(input)), std::istreambuf_iterator()); + CHECK(actual == sentinel); + } + else + { + CHECK_FALSE(fs::exists(destination)); + } + + usize entryCount = 0; + for(const auto& entry : fs::directory_iterator(outputRoot)) + { + ++entryCount; + CHECK(entry.path() == destination); + } + CHECK(entryCount == (existingDestination ? usize{1} : usize{0})); + std::error_code cleanupError; + fs::remove_all(outputRoot, cleanupError); + CHECK_FALSE(cleanupError); + CHECK_FALSE(fs::exists(outputRoot)); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} +} // namespace nx::core::UnitTest::AvizoWriterFailure diff --git a/src/Plugins/SimplnxCore/test/AvizoWriterTest.cpp b/src/Plugins/SimplnxCore/test/AvizoWriterTest.cpp new file mode 100644 index 0000000000..3121ef50c7 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/AvizoWriterTest.cpp @@ -0,0 +1,728 @@ +#include "SimplnxCore/utils/AvizoWriter.hpp" +#include "SimplnxCore/Filters/Algorithms/WriteAvizoRectilinearCoordinate.hpp" +#include "SimplnxCore/Filters/Algorithms/WriteAvizoUniformCoordinate.hpp" +#include "SimplnxCore/SimplnxCore_test_dirs.hpp" + +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/Geometry/ImageGeom.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +/** + * @class ObservedAvizoStore + * @brief Counts source windows and supplies named read diagnostics. + */ +class ObservedAvizoStore : public DataStore +{ +public: + /** + * @brief Initializes one-component feature IDs to seven. + * @param tupleCount Specifies the geometry's X dimension. + */ + explicit ObservedAvizoStore(usize tupleCount) + : DataStore(ShapeType{1, 1, tupleCount}, ShapeType{1}, std::optional{7}) + { + } + + ~ObservedAvizoStore() override = default; + ObservedAvizoStore(const ObservedAvizoStore&) = default; + ObservedAvizoStore(ObservedAvizoStore&&) = default; + ObservedAvizoStore& operator=(const ObservedAvizoStore&) = delete; + ObservedAvizoStore& operator=(ObservedAvizoStore&&) = default; + + /** + * @brief Copies a source window or reports the selected read failure. + * @param startIndex Specifies the first source element. + * @param buffer Receives the feature IDs. + * @return Source data status with the configured warnings or error. + */ + Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + ++readCalls; + if(failReadAt == readCalls) + { + auto result = MakeErrorResult(-92561, "Injected Avizo second-block read failure"); + result.warnings().push_back(Warning{-92574, "Source failure warning"}); + return result; + } + auto result = DataStore::copyIntoBuffer(startIndex, buffer); + if(result.valid() && warnOnRead) + { + result.warnings().push_back(Warning{-92573, "Successful source read warning"}); + } + return result; + } + + usize failReadAt = 0; + bool warnOnRead = false; + mutable usize readCalls = 0; +}; + +/** + * @struct AvizoFixture + * @brief Owns the source data and constructor references for one writer. + */ +struct AvizoFixture +{ + /** + * @brief Creates the geometry, source store, and private output directory. + * @param tupleCount Specifies the geometry's X dimension. + * @param binary Selects the output encoding. + */ + explicit AvizoFixture(usize tupleCount, bool binary) + { + const auto outputBase = std::filesystem::path(unit_test::k_BinaryTestOutputDir.view()); + std::filesystem::create_directories(outputBase); + const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); + outputRoot = outputBase / ("AvizoStreamFailure-" + std::to_string(stamp)); + REQUIRE(std::filesystem::create_directory(outputRoot)); + values.OutputFile = outputRoot / "output.am"; + values.WriteBinaryFile = binary; + values.GeometryPath = DataPath({"ImageGeom"}); + values.FeatureIdsArrayPath = DataPath({"ImageGeom", "CellData", "FeatureIds"}); + values.Units = "microns"; + auto* geom = ImageGeom::Create(dataStructure, "ImageGeom"); + REQUIRE(geom != nullptr); + geom->setDimensions({tupleCount, 1, 1}); + geom->setOrigin({0.0F, 0.0F, 0.0F}); + geom->setSpacing({1.0F, 1.0F, 1.0F}); + auto* cells = AttributeMatrix::Create(dataStructure, "CellData", {1, 1, tupleCount}, geom->getId()); + REQUIRE(cells != nullptr); + geom->setCellData(*cells); + source = std::make_shared(tupleCount); + REQUIRE(Int32Array::Create(dataStructure, "FeatureIds", source, cells->getId()) != nullptr); + } + + /** + * @brief Removes this fixture's output files after each writer is destroyed. + */ + ~AvizoFixture() noexcept + { + std::error_code ignored; + std::filesystem::remove_all(outputRoot, ignored); + } + + AvizoFixture(const AvizoFixture&) = delete; + AvizoFixture(AvizoFixture&&) = delete; + AvizoFixture& operator=(const AvizoFixture&) = delete; + AvizoFixture& operator=(AvizoFixture&&) = delete; + + std::filesystem::path outputRoot; + DataStructure dataStructure; + std::shared_ptr source; + IFilter::MessageHandler handler{[](const IFilter::Message&) {}}; + std::atomic_bool shouldCancel = false; + AvizoWriterInputValues values; +}; + +/** + * @class FaultAvizoWriter + * @brief Supplies per-instance test operations without changing normal stdio dispatch. + * @tparam WriterType Supplies the real algorithm or the stage test writer. + * + * The bundle and its context outlive synchronous execute(). The fixture cannot move. + * Each close callback consumes its real stream before it reports an injected error. + */ +template +class FaultAvizoWriter : public WriterType +{ +public: + /** + * @brief Installs the test-owned operations for this writer only. + * @param dataStructure Owns the image geometry and feature IDs. + * @param handler Receives the writer's messages. + * @param shouldCancel Signals cancellation between windows. + * @param values Specifies output encoding, paths, geometry, and units. + */ + FaultAvizoWriter(DataStructure& dataStructure, const IFilter::MessageHandler& handler, const std::atomic_bool& shouldCancel, AvizoWriterInputValues* values) + : WriterType(dataStructure, handler, shouldCancel, values) + , m_Operations{this, &Write, &Print, &Flush, &Close} + { + this->setOutputOperations(&m_Operations); + } + + /** + * @brief Removes the bundle pointer before the bundle's lifetime ends. + */ + ~FaultAvizoWriter() noexcept override + { + this->setOutputOperations(nullptr); + } + + FaultAvizoWriter(const FaultAvizoWriter&) = delete; + FaultAvizoWriter(FaultAvizoWriter&&) = delete; + FaultAvizoWriter& operator=(const FaultAvizoWriter&) = delete; + FaultAvizoWriter& operator=(FaultAvizoWriter&&) = delete; + + usize failWriteAt = 0; + usize failPrintAt = 0; + std::string_view printFormat; + bool throwFromPrint = false; + bool failFlush = false; + bool failClose = false; + bool throwFromClose = false; + bool rejectEveryWrite = false; + usize writeCalls = 0; + usize printCalls = 0; + usize matchingPrintCalls = 0; + usize flushCalls = 0; + usize closeCalls = 0; + usize observedElementSize = 0; + usize observedRequestedElements = 0; + usize observedWrittenElements = 0; + int nativeCloseStatus = EOF; + +private: + static usize Write(void* context, FILE* stream, const void* data, usize writeElementSize, usize count) + { + auto& self = *static_cast(context); + ++self.writeCalls; + if(self.rejectEveryWrite) + { + // The overflow fixture never transfers its deliberately rejected byte count. + return 0; + } + if(self.writeCalls == self.failWriteAt) + { + self.observedElementSize = writeElementSize; + self.observedRequestedElements = count; + self.observedWrittenElements = std::fwrite(data, writeElementSize, count == 0 ? 0 : count - 1, stream); + errno = ENOSPC; + return self.observedWrittenElements; + } + return std::fwrite(data, writeElementSize, count, stream); + } + + static int Print(void* context, FILE* stream, const char* format, std::va_list args) + { + auto& self = *static_cast(context); + ++self.printCalls; + if(self.printFormat.empty() || self.printFormat == format) + { + ++self.matchingPrintCalls; + if(self.matchingPrintCalls == self.failPrintAt) + { + if(self.throwFromPrint) + { + throw std::runtime_error("Injected variadic print failure"); + } + errno = ENOSPC; + return -1; + } + } + // The callback consumes the caller's va_list once and does not retain it. + return std::vfprintf(stream, format, args); + } + + static int Flush(void* context, FILE* stream) + { + auto& self = *static_cast(context); + ++self.flushCalls; + if(self.failFlush) + { + errno = EIO; + return EOF; + } + return std::fflush(stream); + } + + static int Close(void* context, FILE* stream) + { + auto& self = *static_cast(context); + ++self.closeCalls; + if(self.closeCalls != 1) + { + // A broken owner must produce a count failure without a second access to a closed stream. + errno = EIO; + return EOF; + } + self.nativeCloseStatus = std::fclose(stream); + if(self.throwFromClose) + { + throw std::runtime_error("Injected consumed-stream close failure"); + } + if(self.failClose) + { + errno = EIO; + return EOF; + } + return self.nativeCloseStatus; + } + + typename WriterType::OutputOperations m_Operations; +}; +/** + * @enum StageFault + * @brief Selects a Result or exception at an execution stage. + */ +enum class StageFault +{ + None, ///< Both stages return warnings and success. + HeaderResult, ///< Header returns the named error. + DataResult, ///< Data returns the named error. + HeaderStandard, ///< Header throws a standard exception. + DataStandard, ///< Data throws a standard exception. + HeaderUnknown, ///< Header throws a nonstandard exception. + DataUnknown, ///< Data throws a nonstandard exception. + HeaderAllocation, ///< Header throws an allocation exception. + DataAllocation ///< Data throws an allocation exception. +}; + +/** + * @class StageAvizoWriter + * @brief Supplies stage outcomes without adding production fault controls. + */ +class StageAvizoWriter : public AvizoWriter +{ +public: + using AvizoWriter::AvizoWriter; + + ~StageAvizoWriter() override = default; + StageAvizoWriter(const StageAvizoWriter&) = delete; + StageAvizoWriter(StageAvizoWriter&&) = delete; + StageAvizoWriter& operator=(const StageAvizoWriter&) = delete; + StageAvizoWriter& operator=(StageAvizoWriter&&) = delete; + + StageFault fault = StageFault::None; + mutable usize headerCalls = 0; + mutable usize dataCalls = 0; + +protected: + Result<> generateHeader(FILE*) const override + { + ++headerCalls; + if(fault == StageFault::HeaderStandard) + { + throw std::runtime_error("Injected header exception"); + } + if(fault == StageFault::HeaderUnknown) + { + throw 17; + } + if(fault == StageFault::HeaderAllocation) + { + throw std::bad_alloc{}; + } + auto result = fault == StageFault::HeaderResult ? MakeErrorResult(-92562, "Injected header Result failure") : Result<>{}; + result.warnings().push_back(Warning{-92571, "Header stage warning"}); + return result; + } + + Result<> writeData(FILE*) const override + { + ++dataCalls; + if(fault == StageFault::DataStandard) + { + throw std::runtime_error("Injected data exception"); + } + if(fault == StageFault::DataUnknown) + { + throw 18; + } + if(fault == StageFault::DataAllocation) + { + throw std::bad_alloc{}; + } + auto result = fault == StageFault::DataResult ? MakeErrorResult(-92563, "Injected data Result failure") : Result<>{}; + result.warnings().push_back(Warning{-92572, "Data stage warning"}); + return result; + } +}; + +class OverflowAvizoWriter : public AvizoWriter +{ +public: + using AvizoWriter::AvizoWriter; + + ~OverflowAvizoWriter() override = default; + OverflowAvizoWriter(const OverflowAvizoWriter&) = delete; + OverflowAvizoWriter(OverflowAvizoWriter&&) = delete; + OverflowAvizoWriter& operator=(const OverflowAvizoWriter&) = delete; + OverflowAvizoWriter& operator=(OverflowAvizoWriter&&) = delete; + + usize elementSize = 2; + usize elementCount = std::numeric_limits::max(); + +protected: + Result<> generateHeader(FILE*) const override + { + return {}; + } + + Result<> writeData(FILE* outputFile) const override + { + const char payload = 0; + return writeOutputChecked(outputFile, &payload, elementSize, elementCount); + } +}; + +template +void CheckClosedOnce(const WriterType& writer, const AvizoFixture& fixture) +{ + CHECK(writer.closeCalls == 1); + CHECK(writer.nativeCloseStatus == 0); + std::error_code error; + CHECK(std::filesystem::remove(fixture.values.OutputFile, error)); + CHECK_FALSE(error); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +void CheckOutputFailure(const Result<>& result, const AvizoFixture& fixture) +{ + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().message.find(fixture.values.OutputFile.string()) != std::string::npos); +} + +template +void CheckBinaryWindowFailure(usize failedWindow, bool closeAlsoFails) +{ + AvizoFixture fixture(131073, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.failWriteAt = failedWindow; + writer.failClose = closeAlsoFails; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == (closeAlsoFails ? usize{2} : usize{1})); + const auto& message = result.errors().front().message; + CHECK(message.find(std::to_string(writer.observedRequestedElements)) != std::string::npos); + CHECK(message.find(std::to_string(writer.observedWrittenElements)) != std::string::npos); + CHECK(message.find(std::to_string(writer.observedElementSize)) != std::string::npos); + CHECK(message.find(std::to_string(writer.observedRequestedElements * writer.observedElementSize)) != std::string::npos); + CHECK(message.find(std::to_string(writer.observedWrittenElements * writer.observedElementSize)) != std::string::npos); + CHECK(fixture.source->readCalls == failedWindow); + CHECK(writer.writeCalls == failedWindow); + CHECK(writer.flushCalls == 0); + if(closeAlsoFails) + { + CHECK(result.errors().back().message.find(fixture.values.OutputFile.string()) != std::string::npos); + CHECK(result.errors().back().message.find("close") != std::string::npos); + } + CheckClosedOnce(writer, fixture); +} + +template +void CheckTextFailure(std::string_view format, bool binary, usize expectedReads) +{ + AvizoFixture fixture(65537, binary); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.printFormat = format; + writer.failPrintAt = 1; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + CHECK(writer.matchingPrintCalls == 1); + CHECK(fixture.source->readCalls == expectedReads); + CHECK(writer.flushCalls == 0); + CheckClosedOnce(writer, fixture); +} + +template +void CheckSourceWarnings(bool readFails, bool closeFails) +{ + AvizoFixture fixture(65537, true); + fixture.source->failReadAt = readFails ? 2 : 0; + fixture.source->warnOnRead = true; + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.failClose = closeFails; + const auto result = writer.execute(); + CHECK(result.valid() == (!readFails && !closeFails)); + if(readFails || closeFails) + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == static_cast(readFails) + static_cast(closeFails)); + if(readFails) + { + CHECK(result.errors().front().code == -92561); + } + if(closeFails) + { + CHECK(result.errors().back().message.find("close") != std::string::npos); + } + } + REQUIRE(result.warnings().size() == 2); + CHECK(result.warnings()[0].code == -92573); + CHECK(result.warnings()[1].code == (readFails ? -92574 : -92573)); + CHECK(fixture.source->readCalls == 2); + CHECK(writer.writeCalls == (readFails ? usize{1} : (std::is_same_v ? usize{2} : usize{5}))); + CHECK(writer.flushCalls == (readFails ? usize{0} : usize{1})); + CheckClosedOnce(writer, fixture); +} +} // namespace + +TEST_CASE("SimplnxCore::AvizoWriter: Short binary feature writes stop later reads", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const usize failedWindow = GENERATE(usize{1}, usize{2}, usize{3}); + const bool closeAlsoFails = GENERATE(false, true); + SECTION("Uniform") + { + CheckBinaryWindowFailure(failedWindow, closeAlsoFails); + } + SECTION("Rectilinear") + { + CheckBinaryWindowFailure(failedWindow, closeAlsoFails); + } +} + +TEST_CASE("SimplnxCore::AvizoWriter: Text failures stop production output", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const auto item = GENERATE(std::pair{"# AmiraMesh 3D ASCII 2.0\n", 0}, std::pair{"%d", 1}, std::pair{" ", 1}); + SECTION("Uniform") + { + CheckTextFailure(item.first, false, item.second); + } + SECTION("Rectilinear") + { + CheckTextFailure(item.first, false, item.second); + } +} + +TEST_CASE("SimplnxCore::AvizoWriter: Final feature newline is checked", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool rectilinear = GENERATE(false, true); + const bool writeBinary = GENERATE(false, true); + AvizoFixture fixture(1, writeBinary); + // Both production headers contain one standalone newline before the data newline. + if(rectilinear) + { + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.printFormat = "\n"; + writer.failPrintAt = 2; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + CHECK(writer.writeCalls == (writeBinary ? usize{1} : usize{0})); + CHECK(writer.matchingPrintCalls == 2); + CheckClosedOnce(writer, fixture); + } + else + { + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.printFormat = "\n"; + writer.failPrintAt = 2; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + CHECK(writer.writeCalls == (writeBinary ? usize{1} : usize{0})); + CHECK(writer.matchingPrintCalls == 2); + CheckClosedOnce(writer, fixture); + } + CHECK(fixture.source->readCalls == 1); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Rectilinear coordinate failures stop later axes", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + SECTION("Coordinate delimiter") + { + CheckTextFailure("@2 # x coordinates, then y, then z\n", true, 2); + } + SECTION("Binary coordinate") + { + const usize failedWrite = GENERATE(usize{2}, usize{3}, usize{4}); + AvizoFixture fixture(8, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.failWriteAt = failedWrite; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + CHECK(writer.writeCalls == failedWrite); + CHECK(fixture.source->readCalls == 1); + CHECK(writer.flushCalls == 0); + CheckClosedOnce(writer, fixture); + } + SECTION("Axis newline") + { + const usize failedAxis = GENERATE(usize{1}, usize{2}, usize{3}); + const bool writeBinary = GENERATE(false, true); + AvizoFixture fixture(8, writeBinary); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.printFormat = "\n"; + writer.failPrintAt = 2 + failedAxis; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + CHECK(writer.writeCalls == (writeBinary ? 1 + failedAxis : usize{0})); + CHECK(writer.matchingPrintCalls == 2 + failedAxis); + CheckClosedOnce(writer, fixture); + } + SECTION("ASCII coordinate") + { + CheckTextFailure("%f ", false, 2); + } +} + +TEST_CASE("SimplnxCore::AvizoWriter: Flush and consuming close failures are checked", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool flushFails = GENERATE(false, true); + const bool closeFails = GENERATE(false, true); + AvizoFixture fixture(1, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.failFlush = flushFails; + writer.failClose = closeFails; + const auto result = writer.execute(); + CHECK(writer.headerCalls == 1); + CHECK(writer.dataCalls == 1); + CHECK(writer.flushCalls == 1); + CHECK(result.valid() == (!flushFails && !closeFails)); + if(flushFails || closeFails) + { + CheckOutputFailure(result, fixture); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == static_cast(flushFails) + static_cast(closeFails)); + CHECK(result.errors().front().message.find(flushFails ? "flush" : "close") != std::string::npos); + } + REQUIRE(result.warnings().size() == 2); + CHECK(result.warnings()[0].code == -92571); + CHECK(result.warnings()[1].code == -92572); + CheckClosedOnce(writer, fixture); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Stage Result errors precede consuming close errors", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool headerFails = GENERATE(false, true); + AvizoFixture fixture(1, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.fault = headerFails ? StageFault::HeaderResult : StageFault::DataResult; + writer.failClose = true; + const auto result = writer.execute(); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == (headerFails ? -92562 : -92563)); + CHECK(result.errors()[1].message.find(fixture.values.OutputFile.string()) != std::string::npos); + CHECK(result.errors()[1].message.find("close") != std::string::npos); + CHECK(writer.dataCalls == (headerFails ? usize{0} : usize{1})); + CHECK(writer.flushCalls == 0); + REQUIRE(result.warnings().size() == (headerFails ? usize{1} : usize{2})); + CHECK(result.warnings().front().code == -92571); + CheckClosedOnce(writer, fixture); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Stage exceptions close once", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const auto fault = GENERATE(StageFault::HeaderStandard, StageFault::DataStandard, StageFault::HeaderUnknown, StageFault::DataUnknown); + const bool closeFails = GENERATE(false, true); + AvizoFixture fixture(1, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.fault = fault; + writer.failClose = closeFails; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + REQUIRE(result.invalid()); + CHECK(result.errors().size() == (closeFails ? usize{2} : usize{1})); + const bool dataStage = fault == StageFault::DataStandard || fault == StageFault::DataUnknown; + CHECK(result.errors().front().message.find(dataStage ? "data" : "header") != std::string::npos); + CHECK(writer.dataCalls == (dataStage ? usize{1} : usize{0})); + CHECK(writer.flushCalls == 0); + CHECK(result.warnings().size() == (dataStage ? usize{1} : usize{0})); + CheckClosedOnce(writer, fixture); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Allocation exceptions preserve their type after cleanup", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const auto fault = GENERATE(StageFault::HeaderAllocation, StageFault::DataAllocation); + const bool closeFails = GENERATE(false, true); + AvizoFixture fixture(1, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.fault = fault; + writer.failClose = closeFails; + CHECK_THROWS_AS(writer.execute(), std::bad_alloc); + CHECK(writer.flushCalls == 0); + CheckClosedOnce(writer, fixture); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Callback print exceptions and close exceptions retain ownership", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool printThrows = GENERATE(false, true); + AvizoFixture fixture(1, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + writer.failPrintAt = printThrows ? 1 : 0; + writer.throwFromPrint = printThrows; + writer.throwFromClose = !printThrows; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + CHECK(fixture.source->readCalls == (printThrows ? usize{0} : usize{1})); + CheckClosedOnce(writer, fixture); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Source warnings survive a later read error and close error", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool readFails = GENERATE(false, true); + const bool closeFails = GENERATE(false, true); + SECTION("Uniform") + { + CheckSourceWarnings(readFails, closeFails); + } + SECTION("Rectilinear") + { + CheckSourceWarnings(readFails, closeFails); + } +} + +TEST_CASE("SimplnxCore::AvizoWriter: Faults remain local to one writer", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + AvizoFixture failed(1, true); + AvizoFixture healthy(1, true); + FaultAvizoWriter first(failed.dataStructure, failed.handler, failed.shouldCancel, &failed.values); + FaultAvizoWriter second(healthy.dataStructure, healthy.handler, healthy.shouldCancel, &healthy.values); + first.failWriteAt = 1; + const auto firstResult = first.execute(); + const auto secondResult = second.execute(); + REQUIRE(firstResult.invalid()); + REQUIRE(secondResult.valid()); + CHECK(second.writeCalls == 1); + CHECK(second.flushCalls == 1); + CheckClosedOnce(first, failed); + CheckClosedOnce(second, healthy); +} + +TEST_CASE("SimplnxCore::AvizoWriter: Byte-count overflow stops before the transfer operation", "[SimplnxCore][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool reverseOperands = GENERATE(false, true); + AvizoFixture fixture(1, true); + FaultAvizoWriter writer(fixture.dataStructure, fixture.handler, fixture.shouldCancel, &fixture.values); + if(reverseOperands) + { + std::swap(writer.elementSize, writer.elementCount); + } + // The test operation reports any call without accessing the one-byte source. + writer.rejectEveryWrite = true; + const auto result = writer.execute(); + CheckOutputFailure(result, fixture); + REQUIRE(result.invalid()); + CHECK(result.errors().front().message.find(std::to_string(std::numeric_limits::max())) != std::string::npos); + CHECK(writer.writeCalls == 0); + CHECK(writer.flushCalls == 0); + CheckClosedOnce(writer, fixture); +} diff --git a/src/Plugins/SimplnxCore/test/CMakeLists.txt b/src/Plugins/SimplnxCore/test/CMakeLists.txt index 397fc7e159..54f37717fc 100644 --- a/src/Plugins/SimplnxCore/test/CMakeLists.txt +++ b/src/Plugins/SimplnxCore/test/CMakeLists.txt @@ -14,6 +14,7 @@ set(${PLUGIN_NAME}UnitTest_SRCS AppendImageGeometryTest.cpp ApplyTransformationToGeometryTest.cpp ApproximatePointCloudHullTest.cpp + AvizoWriterTest.cpp ArrayCalculatorTest.cpp ChangeAngleRepresentationTest.cpp CombineAttributeArraysTest.cpp @@ -103,6 +104,8 @@ set(${PLUGIN_NAME}UnitTest_SRCS KeepRemoveRankedFeaturesTest.cpp LabelTriangleGeometryTest.cpp LaplacianSmoothingFilterTest.cpp + LaplacianSmoothingNumericalTest.cpp + LaplacianSmoothingTransferTest.cpp M3CSurfaceMeshingTest.cpp MapPointCloudToRegularGridTest.cpp MoveDataTest.cpp @@ -157,6 +160,7 @@ set(${PLUGIN_NAME}UnitTest_SRCS TriangleDihedralAngleFilterTest.cpp TriangleNormalFilterTest.cpp UncertainRegularGridSampleSurfaceMeshTest.cpp + RepairTriangleWindingTransferTest.cpp VerifyTriangleWindingTest.cpp WriteAbaqusCrystalPlasticityTest.cpp WriteAbaqusHexahedronTest.cpp @@ -179,10 +183,69 @@ set(${PLUGIN_NAME}UnitTest_SRCS create_simplnx_plugin_unit_test(PLUGIN_NAME ${PLUGIN_NAME} FILTER_LIST ${${PLUGIN_NAME}UnitTest_SRCS}) +add_test(NAME SimplnxCoreIdentifySampleBatchCorrectnessTests + COMMAND ${PLUGIN_NAME}UnitTest "[.IdentifySampleBatchCorrectness]" --reporter compact) +set_tests_properties(SimplnxCoreIdentifySampleBatchCorrectnessTests PROPERTIES RUN_SERIAL TRUE TIMEOUT 1500) + +target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/IdentifySampleObservationTest.cpp") +add_test(NAME SimplnxCoreIdentifySampleObservationTests + COMMAND ${PLUGIN_NAME}UnitTest "[.IdentifySampleObservation]" --reporter compact --warn NoTests) +set_tests_properties(SimplnxCoreIdentifySampleObservationTests PROPERTIES RUN_SERIAL TRUE TIMEOUT 300) + +target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/IdentifySampleBatchFailureTest.cpp") +add_test(NAME SimplnxCoreIdentifySampleBatchFailureTests + COMMAND ${PLUGIN_NAME}UnitTest "[.IdentifySampleBatchFailures]" --reporter compact --warn NoTests) +set_tests_properties(SimplnxCoreIdentifySampleBatchFailureTests PROPERTIES RUN_SERIAL TRUE TIMEOUT 300) + +# These headless cases use the normal SimplnxCoreUnitTest runtime and do not require Qt discovery. +option(SIMPLNX_ENABLE_IDENTIFY_SAMPLE_YZ_RESIDENT_BENCHMARKS "Register explicit resident Identify Sample YZ measurements" OFF) +if(SIMPLNX_ENABLE_IDENTIFY_SAMPLE_YZ_RESIDENT_BENCHMARKS) + target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/IdentifySampleYzResidentBenchmark.cpp") + if(NOT "${SIMPLNX_TEST_ALGORITHM_PATH}" STREQUAL "1") + add_test(NAME SimplnxCoreIdentifySampleYzResident_BFS_Bool_NoHoles + COMMAND $ "IdentifySample YZ resident BFS bool noholes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_BFS_Bool_NoHoles PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + add_test(NAME SimplnxCoreIdentifySampleYzResident_BFS_Bool_Holes + COMMAND $ "IdentifySample YZ resident BFS bool holes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_BFS_Bool_Holes PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + add_test(NAME SimplnxCoreIdentifySampleYzResident_BFS_UInt8_NoHoles + COMMAND $ "IdentifySample YZ resident BFS uint8 noholes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_BFS_UInt8_NoHoles PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + add_test(NAME SimplnxCoreIdentifySampleYzResident_BFS_UInt8_Holes + COMMAND $ "IdentifySample YZ resident BFS uint8 holes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_BFS_UInt8_Holes PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + endif() + if(NOT "${SIMPLNX_TEST_ALGORITHM_PATH}" STREQUAL "2") + add_test(NAME SimplnxCoreIdentifySampleYzResident_CCL_Bool_NoHoles + COMMAND $ "IdentifySample YZ resident CCL bool noholes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_CCL_Bool_NoHoles PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + add_test(NAME SimplnxCoreIdentifySampleYzResident_CCL_Bool_Holes + COMMAND $ "IdentifySample YZ resident CCL bool holes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_CCL_Bool_Holes PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + add_test(NAME SimplnxCoreIdentifySampleYzResident_CCL_UInt8_NoHoles + COMMAND $ "IdentifySample YZ resident CCL uint8 noholes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_CCL_UInt8_NoHoles PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + add_test(NAME SimplnxCoreIdentifySampleYzResident_CCL_UInt8_Holes + COMMAND $ "IdentifySample YZ resident CCL uint8 holes" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreIdentifySampleYzResident_CCL_UInt8_Holes PROPERTIES RUN_SERIAL TRUE TIMEOUT 1800 WORKING_DIRECTORY $) + endif() +endif() + if(DREAM3DNX_USE_OOC) add_test(NAME SimplnxCoreOocStoreContracts COMMAND ${PLUGIN_NAME}UnitTest "[.OocStoreContract]" --reporter compact) set_tests_properties(SimplnxCoreOocStoreContracts PROPERTIES RUN_SERIAL TRUE TIMEOUT 1500) + + # One process must retain the OOC resolver after the C8 test changes the default policy. + add_test(NAME SimplnxCoreC8ResolverIsolation + COMMAND ${PLUGIN_NAME}UnitTest + "C8 ReadDREAM3DFilter plans selected data in the destination context,SimplnxCore::ComputeFeatureCentroidsFilter: real HDF5 wrapped weighted oracle" + --order lex --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreC8ResolverIsolation PROPERTIES RUN_SERIAL TRUE TIMEOUT 300) + + add_test(NAME SimplnxCoreLaplacianOocNumerical + COMMAND ${PLUGIN_NAME}UnitTest "[.LaplacianSmoothingOoc]" --reporter compact --warn NoTests) + set_tests_properties(SimplnxCoreLaplacianOocNumerical PROPERTIES RUN_SERIAL TRUE TIMEOUT 300) endif() # ------------------------------------------------------------------------------ @@ -337,3 +400,13 @@ set(PREBUILT_PIPELINE_NAMES # "${${PLUGIN_NAME}_SOURCE_DIR}/pipelines/ReplaceElementAttributesWithNeighbor.d3dpipeline" ) create_pipeline_tests(PLUGIN_NAME ${PLUGIN_NAME} PIPELINE_LIST ${PREBUILT_PIPELINE_NAMES}) + +target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${simplnx_SOURCE_DIR}/test/ProfileIsolationContract.cpp") + +add_test(NAME "${PLUGIN_NAME}_test_profile_isolation" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -DPROFILE_RUNNER_ONLY=TRUE + -P "${simplnx_SOURCE_DIR}/test/VerifyTestProfileLifecycle.cmake") +set_tests_properties("${PLUGIN_NAME}_test_profile_isolation" PROPERTIES RUN_SERIAL TRUE TIMEOUT 60) diff --git a/src/Plugins/SimplnxCore/test/ComputeFeatureCentroidsTest.cpp b/src/Plugins/SimplnxCore/test/ComputeFeatureCentroidsTest.cpp index 1a7ad8d848..ead51b57d2 100644 --- a/src/Plugins/SimplnxCore/test/ComputeFeatureCentroidsTest.cpp +++ b/src/Plugins/SimplnxCore/test/ComputeFeatureCentroidsTest.cpp @@ -1,3 +1,4 @@ +#include "SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.hpp" #include "SimplnxCore/Filters/ComputeFeatureCentroidsFilter.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" @@ -5,6 +6,7 @@ #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/EmptyDataStore.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/Parameters/ArraySelectionParameter.hpp" #include "simplnx/Parameters/BoolParameter.hpp" @@ -12,14 +14,23 @@ #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include +#include #include +#include +#include +#include +#include #include +#include +#include #include #include +#include #include namespace fs = std::filesystem; @@ -79,6 +90,95 @@ class CentroidsFailOnLaterReadStore : public DataStore mutable usize m_ReadCount = 0; }; +class CentroidsObservedReadStore : public DataStore +{ +public: + CentroidsObservedReadStore(const ShapeType& tupleShape, usize failOnRead = 0, std::atomic_bool* cancelAfterRead = nullptr, usize cancelOnRead = 0, usize warnOnRead = 0) + : DataStore(tupleShape, ShapeType{1}, int32{0}) + , m_FailOnRead(failOnRead) + , m_CancelAfterRead(cancelAfterRead) + , m_CancelOnRead(cancelOnRead) + , m_WarnOnRead(warnOnRead) + { + } + + ~CentroidsObservedReadStore() override = default; + CentroidsObservedReadStore(const CentroidsObservedReadStore&) = default; + CentroidsObservedReadStore(CentroidsObservedReadStore&&) = default; + CentroidsObservedReadStore& operator=(const CentroidsObservedReadStore&) = delete; + CentroidsObservedReadStore& operator=(CentroidsObservedReadStore&&) = default; + + Result<> copyIntoBuffer(usize offset, nonstd::span buffer) const override + { + m_ReadCount++; + m_MaxWorkingReserved = std::max(m_MaxWorkingReserved, CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes()); + if(m_ReadCount == m_FailOnRead) + { + return MakeErrorResult(-91912, "Injected periodic centroid read failure"); + } + auto result = DataStore::copyIntoBuffer(offset, buffer); + if(m_ReadCount == m_WarnOnRead) + { + result.warnings().push_back(Warning{-91913, "Injected periodic centroid read warning"}); + } + if(m_CancelAfterRead != nullptr && m_ReadCount == m_CancelOnRead) + { + *m_CancelAfterRead = true; + } + return result; + } + + usize readCount() const + { + return m_ReadCount; + } + + uint64 maxWorkingReserved() const + { + return m_MaxWorkingReserved; + } + +private: + usize m_FailOnRead = 0; + std::atomic_bool* m_CancelAfterRead = nullptr; + usize m_CancelOnRead = 0; + usize m_WarnOnRead = 0; + mutable usize m_ReadCount = 0; + mutable uint64 m_MaxWorkingReserved = 0; +}; + +class CentroidsBudgetSentinel +{ +public: + explicit CentroidsBudgetSentinel(uint64 budgetBytes) + : m_Manager(CacheMemoryBudgetManager::instance()) + , m_PreviousBudget(m_Manager.budgetBytes()) + , m_PreviousReserved(m_Manager.reservedWorkingMemoryBytes()) + { + m_Manager.setBudgetBytes(budgetBytes); + } + + CentroidsBudgetSentinel(const CentroidsBudgetSentinel&) = default; + CentroidsBudgetSentinel(CentroidsBudgetSentinel&&) = default; + CentroidsBudgetSentinel& operator=(const CentroidsBudgetSentinel&) = delete; + CentroidsBudgetSentinel& operator=(CentroidsBudgetSentinel&&) = delete; + + ~CentroidsBudgetSentinel() noexcept + { + m_Manager.setBudgetBytes(m_PreviousBudget); + } + + void requireNoReservationLeak() const + { + REQUIRE(m_Manager.reservedWorkingMemoryBytes() == m_PreviousReserved); + } + +private: + CacheMemoryBudgetManager& m_Manager; + uint64 m_PreviousBudget = 0; + uint64 m_PreviousReserved = 0; +}; + // Build an ImageGeom + Cell Data AM (with FeatureIds) + empty Feature Data AM sized to numFeatures. // featureIds are supplied in row-major (z, y, x) order. inline Scaffold Build(usize dimX, usize dimY, usize dimZ, std::array spacing, std::array origin, usize numFeatures, const std::vector& featureIdValues) @@ -109,16 +209,157 @@ inline Scaffold Build(usize dimX, usize dimY, usize dimZ, std::array return s; } -// Run the filter and return the flat [numFeatures * 3] Centroids values. -inline std::vector Run(Scaffold& s, bool isPeriodic) +inline Arguments MakeArguments(const Scaffold& s, bool isPeriodic) { - ComputeFeatureCentroidsFilter filter; Arguments args; args.insertOrAssign(ComputeFeatureCentroidsFilter::k_SelectedImageGeometryPath_Key, std::make_any(s.geomPath)); args.insertOrAssign(ComputeFeatureCentroidsFilter::k_CellFeatureIdsArrayPath_Key, std::make_any(s.featureIdsPath)); args.insertOrAssign(ComputeFeatureCentroidsFilter::k_FeatureAttributeMatrixPath_Key, std::make_any(s.featureAMPath)); args.insertOrAssign(ComputeFeatureCentroidsFilter::k_CentroidsArrayName_Key, std::make_any(k_CentroidsName)); args.insertOrAssign(ComputeFeatureCentroidsFilter::k_IsPeriodic_Key, std::make_any(isPeriodic)); + return args; +} + +inline std::shared_ptr ObserveIds(Scaffold& s, const std::vector& values, usize failOnRead = 0, std::atomic_bool* cancelAfterRead = nullptr, usize cancelOnRead = 0, + usize warnOnRead = 0) +{ + auto observed = std::make_shared(ShapeType{values.size()}, failOnRead, cancelAfterRead, cancelOnRead, warnOnRead); + for(usize index = 0; index < values.size(); index++) + { + (*observed)[index] = values[index]; + } + REQUIRE_NOTHROW(s.ds.getDataRefAs(s.featureIdsPath)); + auto& array = s.ds.getDataRefAs(s.featureIdsPath); + auto result = array.setDataStore(observed); + SIMPLNX_RESULT_REQUIRE_VALID(result); + return observed; +} + +constexpr float32 k_UnpublishedSentinel = -123.25F; + +inline void PrepareSentinelOutput(Scaffold& s, usize featureCount) +{ + REQUIRE_NOTHROW(s.ds.getDataRefAs(s.featureAMPath)); + const auto& featureAM = s.ds.getDataRefAs(s.featureAMPath); + auto* output = CreateTestDataArray(s.ds, k_CentroidsName, ShapeType{featureCount}, ShapeType{3}, featureAM.getId()); + REQUIRE(output != nullptr); + for(usize index = 0; index < output->getSize(); index++) + { + (*output)[index] = k_UnpublishedSentinel; + } +} + +inline Result<> RunDirect(Scaffold& s, const std::atomic_bool& cancel) +{ + ComputeFeatureCentroidsInputValues inputs; + inputs.FeatureIdsArrayPath = s.featureIdsPath; + inputs.CentroidsArrayPath = s.centroidsPath; + inputs.ImageGeometryPath = s.geomPath; + inputs.FeatureAttributeMatrixPath = s.featureAMPath; + inputs.IsPeriodic = true; + const IFilter::MessageHandler handler{}; + return ComputeFeatureCentroids(s.ds, handler, cancel, &inputs)(); +} + +inline void RequireUnpublished(const Scaffold& s) +{ + REQUIRE_NOTHROW(s.ds.getDataRefAs(s.centroidsPath)); + const auto& output = s.ds.getDataRefAs(s.centroidsPath); + for(usize index = 0; index < output.getSize(); index++) + { + REQUIRE(output[index] == k_UnpublishedSentinel); + } +} + +inline std::vector PopulationIds(const std::vector& populations, usize rows) +{ + std::vector ids(populations.size() * rows, 2); + for(usize y = 0; y < rows; y++) + { + for(usize x = 0; x < populations.size(); x++) + { + if(y < populations[x]) + { + ids[y * populations.size() + x] = 1; + } + } + } + return ids; +} + +// Independent test oracle: sort occupied integer coordinates to find the seam, +// then average every original voxel coordinate so repeated cells retain weight. +inline std::vector SortedCoordinateOracle(const std::array& dims, const std::vector& ids, usize features, const std::array& origin, + const std::array& spacing, bool periodic) +{ + std::vector expected(features * 3, 0.0F); + for(usize feature = 0; feature < features; feature++) + { + for(usize axis = 0; axis < 3; axis++) + { + std::vector weightedCoordinates; + for(usize index = 0; index < ids.size(); index++) + { + if(ids[index] != static_cast(feature)) + { + continue; + } + const std::array q = {index % dims[0], (index / dims[0]) % dims[1], index / (dims[0] * dims[1])}; + weightedCoordinates.push_back(q[axis]); + } + if(weightedCoordinates.empty()) + { + continue; + } + + std::vector occupied = weightedCoordinates; + std::sort(occupied.begin(), occupied.end()); + occupied.erase(std::unique(occupied.begin(), occupied.end()), occupied.end()); + bool adjust = false; + usize cut = 0; + if(periodic && dims[axis] > 1 && occupied.front() == 0 && occupied.back() == dims[axis] - 1) + { + usize largestGap = 0; + bool tied = false; + for(usize index = 0; index < occupied.size(); index++) + { + const usize next = index + 1 == occupied.size() ? occupied.front() + dims[axis] : occupied[index + 1]; + const usize gap = next - occupied[index]; + if(gap > largestGap) + { + largestGap = gap; + cut = occupied[index]; + tied = false; + } + else if(gap == largestGap) + { + tied = true; + } + } + adjust = largestGap > 1 && !tied; + } + + double sum = 0.0; + for(usize q : weightedCoordinates) + { + sum += static_cast(q) + 0.5 + (adjust && q <= cut ? static_cast(dims[axis]) : 0.0); + } + double indexMean = sum / static_cast(weightedCoordinates.size()); + if(adjust) + { + indexMean = std::fmod(indexMean, static_cast(dims[axis])); + } + expected[feature * 3 + axis] = static_cast(static_cast(origin[axis]) + static_cast(spacing[axis]) * indexMean); + } + } + return expected; +} + +// Run the filter and return the flat [numFeatures * 3] Centroids values. +inline std::vector Run(Scaffold& s, bool isPeriodic) +{ + ComputeFeatureCentroidsFilter filter; + Arguments args = MakeArguments(s, isPeriodic); auto preflightResult = filter.preflight(s.ds, args); SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions) @@ -172,6 +413,88 @@ TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: real HDF5 65536-block tai UnitTest::CheckArraysInheritTupleDims(fixture.ds); } +TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: real HDF5 wrapped weighted oracle", "[SimplnxCore][ComputeFeatureCentroidsFilter][.OocStoreContract]") +{ + using namespace CentroidToy; + UnitTest::LoadPlugins(); + REQUIRE(Application::Instance()->getIOManager("HDF5-OOC") != nullptr); + const PreferencesSentinel preferences(DataStorageMode::ForceOutOfCore, 1); + const std::vector ids = {1, 2, 2, 1, 1, 1, 2, 2, 2, 1, 2, 2, 2, 2, 2}; + auto fixture = Build(5, 3, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + REQUIRE_NOTHROW(fixture.ds.getDataRefAs(fixture.featureIdsPath)); + auto& idsArray = fixture.ds.getDataRefAs(fixture.featureIdsPath); + auto store = DataStoreUtilities::ConvertDataStore(idsArray.getDataStoreRef(), "HDF5-OOC"); + REQUIRE(store != nullptr); + auto replaceResult = idsArray.setDataStore(store); + SIMPLNX_RESULT_REQUIRE_VALID(replaceResult); + REQUIRE(idsArray.getDataStoreRef().getDataFormat() == "HDF5-OOC"); + const auto actual = Run(fixture, true); + REQUIRE_NOTHROW(fixture.ds.getDataRefAs(fixture.centroidsPath)); + REQUIRE(fixture.ds.getDataRefAs(fixture.centroidsPath).getDataStoreRef().getDataFormat() == "HDF5-OOC"); + RequireCentroid(actual, 1, 4.7F, 0.9F, 0.5F); + UnitTest::CheckArraysInheritTupleDims(fixture.ds); + + if(std::getenv("SIMPLNX_CENTROID_TIMING") != nullptr) + { + constexpr uint64 k_MiB = 1024ULL * 1024ULL; + constexpr usize k_DimX = 32768; + constexpr usize k_DimY = 16; + constexpr uint64 k_UsefulBytes = k_DimY * (k_DimX / 8) + sizeof(uint64); + std::vector timingIds(k_DimX * k_DimY, static_cast(k_DimY + 1)); + std::vector expected((k_DimY + 2) * 3, 0.0F); + for(usize y = 0; y < k_DimY; y++) + { + timingIds[y * k_DimX] = static_cast(y + 1); + timingIds[y * k_DimX + k_DimX - 1] = static_cast(y + 1); + expected[(y + 1) * 3 + 1] = static_cast(y) + 0.5F; + expected[(y + 1) * 3 + 2] = 0.5F; + } + expected[(k_DimY + 1) * 3] = static_cast(k_DimX) / 2.0F; + expected[(k_DimY + 1) * 3 + 1] = static_cast(k_DimY) / 2.0F; + expected[(k_DimY + 1) * 3 + 2] = 0.5F; + + CentroidsBudgetSentinel budget(256 * k_MiB); + auto& manager = CacheMemoryBudgetManager::instance(); + REQUIRE(manager.reservedWorkingMemoryBytes() == 0); + for(const uint64 headroom : std::array{uint64{8} * k_MiB, uint64{32} * k_MiB, uint64{64} * k_MiB, uint64{64} * 1024ULL, uint64{8} * 1024ULL}) + { + // Populate in memory so setup does not perform one HDF5 write per voxel. + auto timingFixture = [&]() { + const PreferencesSentinel buildResident(DataStorageMode::ForceInCore, 0); + return Build(k_DimX, k_DimY, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, k_DimY + 2, timingIds); + }(); + auto& timingIdsArray = timingFixture.ds.getDataRefAs(timingFixture.featureIdsPath); + REQUIRE(timingIdsArray.getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + auto hdf5Store = DataStoreUtilities::ConvertDataStore(timingIdsArray.getDataStoreRef(), "HDF5-OOC"); + REQUIRE(hdf5Store != nullptr); + auto replaceTimingStoreResult = timingIdsArray.setDataStore(hdf5Store); + SIMPLNX_RESULT_REQUIRE_VALID(replaceTimingStoreResult); + REQUIRE(timingIdsArray.getDataStoreRef().getStoreType() == IDataStore::StoreType::OutOfCore); + REQUIRE(timingIdsArray.getDataStoreRef().getDataFormat() == "HDF5-OOC"); + + const uint64 pressureBytes = 64 * k_MiB - headroom; + auto pressureReservation = manager.reserveWorkingMemory(pressureBytes); + REQUIRE(pressureReservation.sizeBytes() == pressureBytes); + std::vector pressureBuffer(static_cast(pressureBytes), uint8{0}); + const auto started = std::chrono::steady_clock::now(); + const auto timedActual = Run(timingFixture, true); + const auto elapsedMs = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + REQUIRE(timingFixture.ds.getDataRefAs(timingFixture.centroidsPath).getDataStoreRef().getStoreType() == IDataStore::StoreType::OutOfCore); + REQUIRE(timingFixture.ds.getDataRefAs(timingFixture.centroidsPath).getDataStoreRef().getDataFormat() == "HDF5-OOC"); + REQUIRE(timedActual.size() == expected.size()); + for(usize index = 0; index < expected.size(); index++) + { + CAPTURE(headroom, index); + REQUIRE(timedActual[index] == Approx(expected[index]).margin(2.0e-4F)); + } + REQUIRE(manager.reservedWorkingMemoryBytes() == pressureBytes); + const usize plannedOccupancyScans = headroom >= k_UsefulBytes ? 1 : (headroom >= 64 * 1024ULL ? 2 : 9); + std::cout << "CENTROID_HDF5_TIMING headroom=" << headroom << " useful=" << k_UsefulBytes << " planned_occupancy_scans=" << plannedOccupancyScans << " elapsed_ms=" << elapsedMs << '\n'; + } + budget.requireNoReservationLeak(); + } +} + TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: Class 1 - Analytical Centroids", "[SimplnxCore][ComputeFeatureCentroidsFilter]") { using namespace CentroidToy; @@ -215,6 +538,105 @@ TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: Class 1/4 - Periodic Boun using namespace CentroidToy; UnitTest::LoadPlugins(); + SECTION("weighted asymmetric boundary uses largest-gap unwrapped arithmetic") + { + // In a 5x3x1 image, feature 1 occupies x centers 0.5,0.5,3.5,4.5,4.5. + // The unique largest gap is between 0.5 and 3.5; wrapping x=0 forward gives 4.7. + const std::vector ids = {1, 2, 2, 1, 1, 1, 2, 2, 2, 1, 2, 2, 2, 2, 2}; + auto ordinaryFixture = Build(5, 3, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto ordinary = Run(ordinaryFixture, false); + RequireCentroid(ordinary, 1, 2.7F, 0.9F, 0.5F); + + auto periodicFixture = Build(5, 3, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto periodic = Run(periodicFixture, true); + RequireCentroid(periodic, 1, 4.7F, 0.9F, 0.5F); + RequireCentroid(periodic, 0, 0.0F, 0.0F, 0.0F); + UnitTest::CheckArraysInheritTupleDims(periodicFixture.ds); + } + + SECTION("the asymmetric boundary correction works on each axis") + { + for(usize targetAxis = 0; targetAxis < 3; targetAxis++) + { + const usize secondaryAxis = (targetAxis + 1) % 3; + std::array dims = {1, 1, 1}; + dims[targetAxis] = 5; + dims[secondaryAxis] = 3; + std::vector ids(dims[0] * dims[1] * dims[2], 2); + for(usize z = 0; z < dims[2]; z++) + { + for(usize y = 0; y < dims[1]; y++) + { + for(usize x = 0; x < dims[0]; x++) + { + const std::array q = {x, y, z}; + const bool belongs = (q[targetAxis] == 0 || q[targetAxis] == 4) ? q[secondaryAxis] < 2 : q[targetAxis] == 3 && q[secondaryAxis] == 0; + if(belongs) + { + ids[(z * dims[1] + y) * dims[0] + x] = 1; + } + } + } + } + auto fixture = Build(dims[0], dims[1], dims[2], {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto actual = Run(fixture, true); + std::array expected = {0.5F, 0.5F, 0.5F}; + expected[targetAxis] = 4.7F; + expected[secondaryAxis] = 0.9F; + RequireCentroid(actual, 1, expected[0], expected[1], expected[2]); + } + } + + SECTION("origin and signed nonunit spacing map the wrapped index mean") + { + const std::vector ids = {1, 2, 2, 1, 1, 1, 2, 2, 2, 1, 2, 2, 2, 2, 2}; + auto fixture = Build(5, 3, 1, {-2.0F, 3.0F, 0.0F}, {10.0F, -4.0F, 7.0F}, 3, ids); + auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 0.6F, -1.3F, 7.0F); + } + + SECTION("circular-anchor recentering is different from largest-gap arithmetic") + { + const auto ids = PopulationIds({1, 0, 1, 0, 3, 0, 0, 1}, 3); + auto fixture = Build(8, 3, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto actual = Run(fixture, true); + REQUIRE(actual[3] == Approx(8.0F / 3.0F).margin(1.0e-4F)); + } + + SECTION("complete axis occupancy retains the weighted ordinary mean") + { + const auto ids = PopulationIds({3, 1, 1, 1, 1}, 3); + auto fixture = Build(5, 3, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto actual = Run(fixture, true); + REQUIRE(actual[3] == Approx(13.5F / 7.0F).margin(1.0e-4F)); + } + + SECTION("tied largest gaps retain the weighted ordinary mean") + { + const auto ids = PopulationIds({2, 0, 0, 1, 1, 0, 0, 1}, 2); + auto fixture = Build(8, 2, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto actual = Run(fixture, true); + REQUIRE(actual[3] == Approx(3.3F).margin(1.0e-4F)); + } + + SECTION("a feature can wrap on all three axes") + { + std::vector ids(27, 2); + ids[0] = 1; + ids[26] = 1; + auto fixture = Build(3, 3, 3, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 0.0F, 0.0F, 0.0F); + } + + SECTION("populated feature zero is corrected while an unused feature stays zero") + { + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {10.0F, 0.0F, 0.0F}, 3, {0, 1, 1, 1, 0}); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 0, 10.0F, 0.5F, 0.5F); + RequireCentroid(actual, 2, 0.0F, 0.0F, 0.0F); + } + SECTION("Fixture D - periodic wrap, unit spacing (asymmetric)") { // 4x1x1; FeatureIds [1,2,1,1]; feature 1 spans x=0 and x=3 (full extent), feature 2 (x=1) does not. @@ -261,6 +683,308 @@ TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: Class 1/4 - Periodic Boun } } +TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: sorted-coordinate periodic oracle", "[SimplnxCore][ComputeFeatureCentroidsFilter][Periodic]") +{ + using namespace CentroidToy; + UnitTest::LoadPlugins(); + std::mt19937 generator(19077); + std::uniform_int_distribution labels(0, 3); + const std::array origin = {10.0F, -3.0F, 7.0F}; + const std::array spacing = {-2.0F, 1.25F, 0.5F}; + for(usize trial = 0; trial < 48; trial++) + { + const std::array dims = {2 + trial % 5, 2 + (trial / 5) % 4, 2 + (trial / 11) % 3}; + std::vector ids(dims[0] * dims[1] * dims[2]); + for(auto& id : ids) + { + id = labels(generator); + } + INFO("trial " << trial); + auto fixture = Build(dims[0], dims[1], dims[2], spacing, origin, 5, ids); + const auto actual = Run(fixture, true); + const auto expected = SortedCoordinateOracle(dims, ids, 5, origin, spacing, true); + REQUIRE(actual.size() == expected.size()); + for(usize index = 0; index < expected.size(); index++) + { + REQUIRE(actual[index] == Approx(expected[index]).margin(2.0e-4F)); + } + } +} + +TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: bounded periodic scans and failures", "[SimplnxCore][ComputeFeatureCentroidsFilter][Periodic]") +{ + using namespace CentroidToy; + UnitTest::LoadPlugins(); + const PreferencesSentinel resident(DataStorageMode::ForceInCore, 0); + + SECTION("no wrapped pair avoids both occupancy and correction scans") + { + CentroidsBudgetSentinel budget(0); + const std::vector ids = {2, 1, 1, 1, 3}; + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 4, ids); + auto observed = ObserveIds(fixture, ids); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 2.5F, 0.5F, 0.5F); + REQUIRE(observed->readCount() == 2); // validator + original algorithm scan + budget.requireNoReservationLeak(); + } + + SECTION("one wrapped pair has one occupancy scan and one weighted scan") + { + const std::vector ids = {1, 2, 2, 2, 1}; + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto observed = ObserveIds(fixture, ids); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 0.0F, 0.5F, 0.5F); + REQUIRE(observed->readCount() == 4); // validator + original + occupancy + weighted + } + + SECTION("tied or full-axis gaps omit the weighted correction scan") + { + const std::vector ids = {1, 1, 1, 1, 1}; + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 2, ids); + auto observed = ObserveIds(fixture, ids); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 2.5F, 0.5F, 0.5F); + REQUIRE(observed->readCount() == 3); + } + + SECTION("two wrapped features batch into one or two occupancy scans") + { + const std::vector ids = {1, 3, 3, 3, 3, 3, 3, 1, 2, 4, 4, 4, 4, 4, 4, 2}; + std::vector roomy; + { + CentroidsBudgetSentinel budget(64); // one quarter gives 16 occupancy bytes + auto fixture = Build(8, 2, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 5, ids); + auto observed = ObserveIds(fixture, ids); + roomy = Run(fixture, true); + REQUIRE(observed->readCount() == 4); + budget.requireNoReservationLeak(); + } + { + CentroidsBudgetSentinel budget(32); // one quarter gives one 8-byte word + auto fixture = Build(8, 2, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 5, ids); + auto observed = ObserveIds(fixture, ids); + const auto limited = Run(fixture, true); + REQUIRE(limited == roomy); + REQUIRE(observed->readCount() == 5); + budget.requireNoReservationLeak(); + } + } + + SECTION("minimum grant carries a unique gap across empty windows") + { + CentroidsBudgetSentinel budget(32); + std::vector ids(260, 2); + ids[0] = 1; + ids[1] = 1; + ids[258] = 1; + ids[259] = 1; + auto fixture = Build(260, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto observed = ObserveIds(fixture, ids); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 0.0F, 0.5F, 0.5F); + REQUIRE(observed->readCount() == 8); // validator + original + five windows + weighted + budget.requireNoReservationLeak(); + } + + SECTION("word 63/64 boundary and partial last word preserve the unique cut") + { + CentroidsBudgetSentinel budget(32); + std::vector ids(130, 2); + for(usize q : {usize{0}, usize{63}, usize{64}, usize{129}}) + { + ids[q] = 1; + } + auto fixture = Build(130, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto observed = ObserveIds(fixture, ids); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 32.0F, 0.5F, 0.5F); + REQUIRE(observed->readCount() == 6); // validator + original + three windows + weighted + budget.requireNoReservationLeak(); + } + + SECTION("equal maxima across windows retain the ordinary mean") + { + CentroidsBudgetSentinel budget(32); + std::vector ids(129, 2); + for(usize q : {usize{0}, usize{64}, usize{128}}) + { + ids[q] = 1; + } + auto fixture = Build(129, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto observed = ObserveIds(fixture, ids); + const auto actual = Run(fixture, true); + RequireCentroid(actual, 1, 64.5F, 0.5F, 0.5F); + REQUIRE(observed->readCount() == 5); // validator + original + three windows, no weighted pass + budget.requireNoReservationLeak(); + } + + SECTION("subminimum and zero grants fail before output publication") + { + for(uint64 budgetBytes : {uint64{28}, uint64{0}}) + { + const std::vector ids = {1, 2, 2, 1, 1}; + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + PrepareSentinelOutput(fixture, 3); + CentroidsBudgetSentinel budget(budgetBytes); + const std::atomic_bool cancel = false; + const auto result = RunDirect(fixture, cancel); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().front().code == -5321); + RequireUnpublished(fixture); + budget.requireNoReservationLeak(); + } + } + + SECTION("occupancy and weighted read failures leave the final output unpublished") + { + const std::vector ids = {1, 2, 2, 1, 1}; + for(usize failOnRead : {usize{3}, usize{4}}) + { + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + PrepareSentinelOutput(fixture, 3); + auto observed = ObserveIds(fixture, ids, failOnRead, nullptr, 0, failOnRead == 4 ? usize{3} : usize{0}); + const uint64 previousReservation = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + const std::atomic_bool cancel = false; + const auto result = RunDirect(fixture, cancel); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().front().code == -91912); + REQUIRE(observed->readCount() == failOnRead); + REQUIRE(result.warnings().size() == (failOnRead == 4 ? 1 : 0)); + RequireUnpublished(fixture); + REQUIRE(CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes() == previousReservation); + } + } + + SECTION("cancellation in occupancy or weighted pass leaves the final output unpublished") + { + const std::vector ids = {1, 2, 2, 1, 1}; + for(usize cancelOnRead : {usize{3}, usize{4}}) + { + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + PrepareSentinelOutput(fixture, 3); + std::atomic_bool cancel = false; + auto observed = ObserveIds(fixture, ids, 0, &cancel, cancelOnRead, cancelOnRead); + const uint64 previousReservation = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + const auto result = RunDirect(fixture, cancel); + REQUIRE(cancel); + REQUIRE(observed->readCount() == cancelOnRead); + REQUIRE(result.warnings().size() == 1); + RequireUnpublished(fixture); + REQUIRE(CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes() == previousReservation); + } + } + + SECTION("successful occupancy and weighted read warnings reach the caller") + { + const std::vector ids = {1, 2, 2, 1, 1}; + for(usize warnOnRead : {usize{3}, usize{4}}) + { + auto fixture = Build(5, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 3, ids); + auto observed = ObserveIds(fixture, ids, 0, nullptr, 0, warnOnRead); + ComputeFeatureCentroidsFilter filter; + const auto executeResult = filter.execute(fixture.ds, MakeArguments(fixture, true)); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + REQUIRE(executeResult.result.warnings().size() == 1); + REQUIRE(executeResult.result.warnings().front().code == -91913); + REQUIRE(observed->readCount() == 4); + REQUIRE_NOTHROW(fixture.ds.getDataRefAs(fixture.centroidsPath)); + REQUIRE(fixture.ds.getDataRefAs(fixture.centroidsPath)[3] == Approx(4.5F)); + } + } + + SECTION("opt-in bounded grant timing probe") + { + if(std::getenv("SIMPLNX_CENTROID_TIMING") != nullptr) + { + constexpr uint64 k_MiB = 1024ULL * 1024ULL; + constexpr usize k_DimX = 32768; + constexpr usize k_DimY = 16; + constexpr uint64 k_UsefulBytes = k_DimY * (k_DimX / 8) + sizeof(uint64); // 16 X pairs plus one full-Y background pair + std::vector ids(k_DimX * k_DimY, static_cast(k_DimY + 1)); + for(usize y = 0; y < k_DimY; y++) + { + ids[y * k_DimX] = static_cast(y + 1); + ids[y * k_DimX + k_DimX - 1] = static_cast(y + 1); + } + CentroidsBudgetSentinel budget(256 * k_MiB); // fixed cache budget, 64 MiB maximum working allowance + auto& manager = CacheMemoryBudgetManager::instance(); + REQUIRE(manager.reservedWorkingMemoryBytes() == 0); + std::vector reference; + for(const uint64 headroom : std::array{uint64{8} * k_MiB, uint64{32} * k_MiB, uint64{64} * k_MiB, uint64{64} * 1024ULL, uint64{8} * 1024ULL}) + { + const uint64 pressureBytes = 64 * k_MiB - headroom; + auto pressureReservation = manager.reserveWorkingMemory(pressureBytes); + REQUIRE(pressureReservation.sizeBytes() == pressureBytes); + std::vector pressureBuffer(static_cast(pressureBytes), uint8{0}); + REQUIRE(pressureBuffer.size() == pressureBytes); + auto fixture = Build(k_DimX, k_DimY, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, k_DimY + 2, ids); + auto observed = ObserveIds(fixture, ids); + const auto started = std::chrono::steady_clock::now(); + const auto actual = Run(fixture, true); + const auto elapsedMs = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + const uint64 granted = observed->maxWorkingReserved() - pressureReservation.sizeBytes(); + const usize occupancyScans = (observed->readCount() - 14 - 8 - 8) / 8; + const usize expectedScans = headroom >= k_UsefulBytes ? 1 : (headroom >= 64 * 1024ULL ? 2 : 9); + REQUIRE(granted == std::min(k_UsefulBytes, headroom)); + REQUIRE(occupancyScans == expectedScans); + REQUIRE(observed->readCount() == 14 + (occupancyScans + 2) * 8); + RequireCentroid(actual, 1, 0.0F, 0.5F, 0.5F); + if(reference.empty()) + { + reference = actual; + } + else + { + REQUIRE(actual == reference); + } + std::cout << "CENTROID_TIMING headroom=" << headroom << " useful=" << k_UsefulBytes << " grant=" << granted << " occupancy_scans=" << occupancyScans + << " logical_reads=" << observed->readCount() << " elapsed_ms=" << elapsedMs << '\n'; + } + budget.requireNoReservationLeak(); + } + } +} + +TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: checked working sizes", "[SimplnxCore][ComputeFeatureCentroidsFilter]") +{ + using namespace CentroidToy; + UnitTest::LoadPlugins(); + const PreferencesSentinel resident(DataStorageMode::ForceInCore, 0); + const std::atomic_bool cancel = false; + + SECTION("feature-sized byte multiplication fails before allocating scratch") + { + constexpr usize k_HugeFeatureCount = std::numeric_limits::max() / (3 * sizeof(uint64)) + 1; + auto fixture = Build(1, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, k_HugeFeatureCount, {0}); + auto plannedStore = EmptyDataStore::Create(ShapeType{k_HugeFeatureCount}, ShapeType{3}, ""); + SIMPLNX_RESULT_REQUIRE_VALID(plannedStore); + REQUIRE_NOTHROW(fixture.ds.getDataRefAs(fixture.featureAMPath)); + const auto& featureAM = fixture.ds.getDataRefAs(fixture.featureAMPath); + auto* output = Float32Array::Create(fixture.ds, k_CentroidsName, std::shared_ptr>(std::move(plannedStore.value())), featureAM.getId()); + REQUIRE(output != nullptr); + const auto result = RunDirect(fixture, cancel); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().front().code == -5320); + } + + SECTION("geometry XY and XYZ products fail before voxel scans") + { + for(const SizeVec3& dimensions : {SizeVec3{std::numeric_limits::max(), 2, 1}, SizeVec3{std::numeric_limits::max() / 2, 2, 2}}) + { + auto fixture = Build(1, 1, 1, {1.0F, 1.0F, 1.0F}, {0.0F, 0.0F, 0.0F}, 2, {0}); + PrepareSentinelOutput(fixture, 2); + REQUIRE_NOTHROW(fixture.ds.getDataRefAs(fixture.geomPath)); + fixture.ds.getDataRefAs(fixture.geomPath).setDimensions(dimensions); + const auto result = RunDirect(fixture, cancel); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().front().code == -5320); + RequireUnpublished(fixture); + } + } +} + TEST_CASE("SimplnxCore::ComputeFeatureCentroidsFilter: Error - FeatureId exceeds Feature AM", "[SimplnxCore][ComputeFeatureCentroidsFilter]") { using namespace CentroidToy; diff --git a/src/Plugins/SimplnxCore/test/ComputeKMeansTest.cpp b/src/Plugins/SimplnxCore/test/ComputeKMeansTest.cpp index 4974efe501..21c0ebd4bd 100644 --- a/src/Plugins/SimplnxCore/test/ComputeKMeansTest.cpp +++ b/src/Plugins/SimplnxCore/test/ComputeKMeansTest.cpp @@ -2,6 +2,7 @@ #include #include +#include "simplnx/Common/Types.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" @@ -14,8 +15,12 @@ #include "simplnx/Utilities/AlgorithmDispatch.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" +#include "SimplnxCore/Filters/Algorithms/ComputeKMeans.hpp" +#include "SimplnxCore/Filters/Algorithms/ComputeKMeansDirect.hpp" +#include "SimplnxCore/Filters/Algorithms/ComputeKMeansScanline.hpp" #include "SimplnxCore/Filters/ComputeKMeansFilter.hpp" +#include #include #include #include @@ -180,6 +185,45 @@ void BuildKMeansFixture(DataStructure& dataStructure, const std::vector& valu } } +/** + * @brief Builds three scalar tuples whose mean is four. + * @param dataStructure Receives the geometry and input array. + * @param useOocStore True to create the input with the selected OOC store. + */ +void BuildKMeansOneClusterData(DataStructure& dataStructure, bool useOocStore) +{ + const ShapeType tupleShape = {3, 1, 1}; + auto* imageGeom = ImageGeom::Create(dataStructure, k_MaskParityGeometryPath.getTargetName()); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({3, 1, 1}); + auto* cellData = AttributeMatrix::Create(dataStructure, k_MaskParityCellDataPath.getTargetName(), tupleShape, imageGeom->getId()); + REQUIRE(cellData != nullptr); + imageGeom->setCellData(*cellData); + + auto inputStore = CreateKMeansStore(dataStructure, k_MaskParityInputPath, tupleShape, {1}, useOocStore); + REQUIRE(Float32Array::Create(dataStructure, k_MaskParityInputPath.getTargetName(), inputStore, cellData->getId()) != nullptr); + const std::array inputValues = {2.0F, 4.0F, 6.0F}; + auto writeResult = inputStore->copyFromBuffer(0, nonstd::span(inputValues.data(), inputValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); +} + +/** + * @brief Creates arguments with a fixed seed, no mask, and the filter's cluster default. + * @return Arguments for the three-tuple one-cluster fixture. + */ +Arguments CreateKMeansOneClusterArguments() +{ + ComputeKMeansFilter filter; + Arguments args = filter.getDefaultArguments(); + args.insertOrAssign(ComputeKMeansFilter::k_UseSeed_Key, std::make_any(true)); + args.insertOrAssign(ComputeKMeansFilter::k_SeedValue_Key, std::make_any(5489)); + args.insertOrAssign(ComputeKMeansFilter::k_SelectedArrayPath_Key, std::make_any(k_MaskParityInputPath)); + args.insertOrAssign(ComputeKMeansFilter::k_FeatureIdsArrayName_Key, std::make_any(k_MaskParityIdsPath.getTargetName())); + args.insertOrAssign(ComputeKMeansFilter::k_FeatureAMPath_Key, std::make_any(k_MaskParityFeatureDataPath)); + args.insertOrAssign(ComputeKMeansFilter::k_MeansArrayName_Key, std::make_any(k_MaskParityMeansPath.getTargetName())); + return args; +} + /** * @brief Reads all values from one K-means output array. * @tparam T Specifies the array element type. @@ -561,6 +605,104 @@ TEST_CASE("SimplnxCore::ComputeKMeans: Scanline rejects an all-false mask", "[Si UnitTest::CheckArraysInheritTupleDims(dataStructure); } +TEST_CASE("SimplnxCore::ComputeKMeans: default one cluster", "[SimplnxCore][ComputeKMeans]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + + DataStructure dataStructure; + BuildKMeansOneClusterData(dataStructure, false); + ComputeKMeansFilter filter; + Arguments args = CreateKMeansOneClusterArguments(); + REQUIRE(args.value(ComputeKMeansFilter::k_InitClusters_Key) == 1); + REQUIRE(args.value(ComputeKMeansFilter::k_UseMask_Key) == false); + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + CHECK(ReadKMeansValues(dataStructure, k_MaskParityIdsPath) == std::vector{1, 1, 1}); + CHECK(ReadKMeansValues(dataStructure, k_MaskParityMeansPath) == std::vector{0.0, 4.0}); + const auto seedPath = DataPath({args.value(ComputeKMeansFilter::k_SeedArrayName_Key)}); + CHECK(ReadKMeansValues(dataStructure, seedPath) == std::vector{5489}); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("SimplnxCore::ComputeKMeans: zero clusters fail before output actions", "[SimplnxCore][ComputeKMeans]") +{ + UnitTest::LoadPlugins(); + DataStructure dataStructure; + BuildKMeansOneClusterData(dataStructure, false); + ComputeKMeansFilter filter; + Arguments args = CreateKMeansOneClusterArguments(); + args.insertOrAssign(ComputeKMeansFilter::k_InitClusters_Key, std::make_any(0)); + + const auto preflightResult = filter.preflight(dataStructure, args); + REQUIRE(preflightResult.outputActions.invalid()); + REQUIRE(preflightResult.outputActions.errors().front().code == -54061); + CHECK(preflightResult.outputActions.errors().front().message.find("0") != std::string::npos); + CHECK(preflightResult.outputActions.errors().front().message.find(k_MaskParityInputPath.toString()) != std::string::npos); + REQUIRE(dataStructure.getData(k_MaskParityIdsPath) == nullptr); + REQUIRE(dataStructure.getData(k_MaskParityFeatureDataPath) == nullptr); + + const auto executeResult = filter.execute(dataStructure, args); + REQUIRE(executeResult.result.invalid()); + REQUIRE(executeResult.result.errors().front().code == -54061); + CHECK(dataStructure.getData(k_MaskParityIdsPath) == nullptr); + CHECK(dataStructure.getData(k_MaskParityFeatureDataPath) == nullptr); + CHECK(dataStructure.getData(DataPath({args.value(ComputeKMeansFilter::k_SeedArrayName_Key)})) == nullptr); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("SimplnxCore::ComputeKMeans: zero clusters leave direct and scanline outputs unchanged", "[SimplnxCore][ComputeKMeans]") +{ + UnitTest::LoadPlugins(); + for(const bool useScanline : {false, true}) + { + for(const bool cancelled : {false, true}) + { + DYNAMIC_SECTION("scanline=" << useScanline << " cancelled=" << cancelled) + { + DataStructure dataStructure; + BuildKMeansOneClusterData(dataStructure, false); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_MaskParityGeometryPath)); + const auto& imageGeom = dataStructure.getDataRefAs(k_MaskParityGeometryPath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_MaskParityCellDataPath)); + const auto& cellData = dataStructure.getDataRefAs(k_MaskParityCellDataPath); + auto idStore = std::make_shared>(ShapeType{3, 1, 1}, ShapeType{1}, int32{-77}); + REQUIRE(Int32Array::Create(dataStructure, k_MaskParityIdsPath.getTargetName(), idStore, cellData.getId()) != nullptr); + auto* featureData = AttributeMatrix::Create(dataStructure, k_MaskParityFeatureDataPath.getTargetName(), ShapeType{1}, imageGeom.getId()); + REQUIRE(featureData != nullptr); + auto meanStore = std::make_shared>(ShapeType{1}, ShapeType{1}, 91.0F); + REQUIRE(Float32Array::Create(dataStructure, k_MaskParityMeansPath.getTargetName(), meanStore, featureData->getId()) != nullptr); + + ComputeKMeansInputValues values{}; + values.InitClusters = 0; + values.DistanceMetric = ClusterUtilities::DistanceMetric::Euclidean; + values.UseMask = true; + values.ClusteringArrayPath = k_MaskParityInputPath; + // The mask does not exist. The cluster count must fail before mask access. + values.MaskArrayPath = k_MaskParityMaskPath; + values.FeatureIdsArrayPath = k_MaskParityIdsPath; + values.MeansArrayPath = k_MaskParityMeansPath; + values.Seed = 5489; + const std::atomic_bool shouldCancel = cancelled; + const auto result = useScanline ? ComputeKMeansScanline(dataStructure, IFilter::MessageHandler{}, shouldCancel, &values)() : + ComputeKMeansDirect(dataStructure, IFilter::MessageHandler{}, shouldCancel, &values)(); + REQUIRE(result.invalid()); + REQUIRE(result.errors().front().code == -54061); + CHECK(result.errors().front().message.find("0") != std::string::npos); + CHECK(result.errors().front().message.find(k_MaskParityInputPath.toString()) != std::string::npos); + CHECK(ReadKMeansValues(dataStructure, k_MaskParityIdsPath) == std::vector{-77, -77, -77}); + CHECK(ReadKMeansValues(dataStructure, k_MaskParityMeansPath) == std::vector{91.0F}); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + } + } +} + TEST_CASE("SimplnxCore::ComputeKMeansFilter: SIMPL Backwards Compatibility", "[SimplnxCore][ComputeKMeansFilter][BackwardsCompatibility]") { auto app = Application::GetOrCreateInstance(); diff --git a/src/Plugins/SimplnxCore/test/ComputeSurfaceAreaToVolumeTest.cpp b/src/Plugins/SimplnxCore/test/ComputeSurfaceAreaToVolumeTest.cpp index 9710f541b8..f69f7f5bbc 100644 --- a/src/Plugins/SimplnxCore/test/ComputeSurfaceAreaToVolumeTest.cpp +++ b/src/Plugins/SimplnxCore/test/ComputeSurfaceAreaToVolumeTest.cpp @@ -20,11 +20,8 @@ #include "simplnx/DataStructure/StringArray.hpp" #include "simplnx/Filter/Arguments.hpp" #include "simplnx/Filter/IFilter.hpp" -#include "simplnx/Parameters/ArrayCreationParameter.hpp" -#include "simplnx/Parameters/BoolParameter.hpp" #include "simplnx/UnitTest/AlgorithmTestScope.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" -#include "simplnx/Utilities/AlgorithmDispatch.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include @@ -38,6 +35,8 @@ #include #include #include +#include +#include #include #include #include @@ -677,3 +676,160 @@ TEST_CASE("SimplnxCore::ComputeSurfaceAreaToVolumeFilter: 200x200x200 octant fea UnitTest::CheckArraysInheritTupleDims(dataStructure); } + +namespace +{ +const DataPath k_MetricGeometryPath({"Surface Metrics"}); +const DataPath k_MetricCellPath = k_MetricGeometryPath.createChildPath("Cell Data"); +const DataPath k_MetricFeaturePath = k_MetricGeometryPath.createChildPath("Feature Data"); +const DataPath k_MetricIdsPath = k_MetricCellPath.createChildPath("Feature Ids"); +const DataPath k_MetricCountsPath = k_MetricFeaturePath.createChildPath("Cell Counts"); +const DataPath k_MetricRatioPath = k_MetricFeaturePath.createChildPath("Analytical Ratio"); +const DataPath k_MetricSphericityPath = k_MetricFeaturePath.createChildPath("Analytical Sphericity"); + +Arguments BuildSurfaceMetricFixture(DataStructure& dataStructure, const FloatVec3& spacing, const std::vector& featureIds, const std::vector& counts, bool calculateSphericity) +{ + REQUIRE(featureIds.size() == 125); + auto* imageGeom = ImageGeom::Create(dataStructure, k_MetricGeometryPath.getTargetName()); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({5, 5, 5}); + imageGeom->setOrigin({0.0f, 0.0f, 0.0f}); + imageGeom->setSpacing(spacing); + auto* cellData = AttributeMatrix::Create(dataStructure, k_MetricCellPath.getTargetName(), {5, 5, 5}, imageGeom->getId()); + REQUIRE(cellData != nullptr); + imageGeom->setCellData(*cellData); + auto* featureData = AttributeMatrix::Create(dataStructure, k_MetricFeaturePath.getTargetName(), {counts.size()}, imageGeom->getId()); + REQUIRE(featureData != nullptr); + auto idsStore = DataStoreUtilities::CreateDataStore(dataStructure, k_MetricIdsPath, {5, 5, 5}, {1}); + auto countsStore = DataStoreUtilities::CreateDataStore(dataStructure, k_MetricCountsPath, {counts.size()}, {1}); + REQUIRE(idsStore != nullptr); + REQUIRE(countsStore != nullptr); + REQUIRE(Int32Array::Create(dataStructure, k_MetricIdsPath.getTargetName(), idsStore, cellData->getId()) != nullptr); + REQUIRE(Int32Array::Create(dataStructure, k_MetricCountsPath.getTargetName(), countsStore, featureData->getId()) != nullptr); + const auto idsWrite = idsStore->copyFromBuffer(0, nonstd::span(featureIds.data(), featureIds.size())); + SIMPLNX_RESULT_REQUIRE_VALID(idsWrite); + const auto countsWrite = countsStore->copyFromBuffer(0, nonstd::span(counts.data(), counts.size())); + SIMPLNX_RESULT_REQUIRE_VALID(countsWrite); + + auto args = ComputeSurfaceAreaToVolumeFilter().getDefaultArguments(); + args.insertOrAssign(ComputeSurfaceAreaToVolumeFilter::k_SelectedImageGeometryPath_Key, std::make_any(k_MetricGeometryPath)); + args.insertOrAssign(ComputeSurfaceAreaToVolumeFilter::k_CellFeatureIdsArrayPath_Key, std::make_any(k_MetricIdsPath)); + args.insertOrAssign(ComputeSurfaceAreaToVolumeFilter::k_NumCellsArrayPath_Key, std::make_any(k_MetricCountsPath)); + args.insertOrAssign(ComputeSurfaceAreaToVolumeFilter::k_CalculateSphericity_Key, std::make_any(calculateSphericity)); + args.insertOrAssign(ComputeSurfaceAreaToVolumeFilter::k_SurfaceAreaVolumeRatioArrayName_Key, std::make_any(k_MetricRatioPath.getTargetName())); + args.insertOrAssign(ComputeSurfaceAreaToVolumeFilter::k_SphericityArrayName_Key, std::make_any(k_MetricSphericityPath.getTargetName())); + return args; +} + +template +std::vector ReadSurfaceMetricArray(DataStructure& dataStructure, const DataPath& path, const ShapeType& tupleShape, const UnitTest::AlgorithmTestScope& scope) +{ + CAPTURE(path.toString()); + const DataArray* array = nullptr; + REQUIRE_NOTHROW(array = &dataStructure.getDataRefAs>(path)); + scope.requireExpectedStore(*array); + REQUIRE(array->getTupleShape() == tupleShape); + REQUIRE(array->getComponentShape() == ShapeType{1}); + REQUIRE(array->getDataType() == GetDataType()); + const AttributeMatrix* parent = nullptr; + REQUIRE_NOTHROW(parent = &dataStructure.getDataRefAs(path.getParent())); + REQUIRE(array->getParentIds().size() == 1); + REQUIRE(array->getParentIds().front() == parent->getId()); + const auto& store = array->getDataStoreRef(); + + std::vector values(array->getSize()); + const auto readResult = store.copyIntoBuffer(0, nonstd::span(values.data(), values.size())); + SIMPLNX_RESULT_REQUIRE_VALID(readResult); + return values; +} + +void RequireSurfaceMetricGeometry(DataStructure& dataStructure, const ImageGeom& imageGeom, const FloatVec3& spacing, usize featureCount) +{ + const SizeVec3 dimensions{5, 5, 5}; + const FloatVec3 origin{0.0f, 0.0f, 0.0f}; + REQUIRE(dataStructure.getData(k_MetricGeometryPath) == &imageGeom); + REQUIRE(imageGeom.getDimensions() == dimensions); + REQUIRE(imageGeom.getSpacing() == spacing); + REQUIRE(imageGeom.getOrigin() == origin); + REQUIRE(imageGeom.getCellDataPath() == k_MetricCellPath); + const AttributeMatrix* cellData = nullptr; + const AttributeMatrix* featureData = nullptr; + REQUIRE_NOTHROW(cellData = &dataStructure.getDataRefAs(k_MetricCellPath)); + REQUIRE_NOTHROW(featureData = &dataStructure.getDataRefAs(k_MetricFeaturePath)); + REQUIRE(imageGeom.getCellData() == cellData); + const ShapeType cellShape{5, 5, 5}; + REQUIRE(cellData->getShape() == cellShape); + REQUIRE(featureData->getShape() == ShapeType{featureCount}); + REQUIRE(cellData->getParentIds().size() == 1); + REQUIRE(cellData->getParentIds().front() == imageGeom.getId()); + REQUIRE(featureData->getParentIds().size() == 1); + REQUIRE(featureData->getParentIds().front() == imageGeom.getId()); +} +} // namespace + +TEST_CASE("SimplnxCore::ComputeSurfaceAreaToVolumeFilter: analytical interior features", "[SimplnxCore][ComputeSurfaceAreaToVolumeFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool calculateSphericity = GENERATE(false, true); + DYNAMIC_SECTION(scenario << ", sphericity=" << calculateSphericity) + { + UnitTest::AlgorithmTestScope scope(scenario); + DataStructure dataStructure; + const FloatVec3 spacing{1.0f, 1.0f, 1.0f}; + std::vector featureIds(125, 1); + // A positive enclosing feature separates the interior shapes from the image boundary. + featureIds[31] = 2; + featureIds[92] = 3; + featureIds[93] = 3; + const std::vector counts{0, 122, 1, 2}; + const auto args = BuildSurfaceMetricFixture(dataStructure, spacing, featureIds, counts, calculateSphericity); + const ImageGeom* imageGeom = nullptr; + REQUIRE_NOTHROW(imageGeom = &dataStructure.getDataRefAs(k_MetricGeometryPath)); + REQUIRE(imageGeom->getIndex(1.5f, 1.5f, 1.5f) == std::optional{31}); + REQUIRE(imageGeom->getIndex(2.5f, 3.5f, 3.5f) == std::optional{92}); + REQUIRE(imageGeom->getIndex(3.5f, 3.5f, 3.5f) == std::optional{93}); + REQUIRE(ReadSurfaceMetricArray(dataStructure, k_MetricIdsPath, {5, 5, 5}, scope) == featureIds); + REQUIRE(ReadSurfaceMetricArray(dataStructure, k_MetricCountsPath, {4}, scope) == counts); + + ComputeSurfaceAreaToVolumeFilter filter; + const auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + REQUIRE(dataStructure.getData(k_MetricRatioPath) == nullptr); + REQUIRE(dataStructure.getData(k_MetricSphericityPath) == nullptr); + const auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + const auto ratios = ReadSurfaceMetricArray(dataStructure, k_MetricRatioPath, {4}, scope); + REQUIRE(ratios.size() == 4); + REQUIRE(ratios[0] == 0.0f); + // The enclosing feature has 16 internal faces; the filter omits its exterior faces. + REQUIRE(std::abs(static_cast(ratios[1]) - 8.0 / 61.0) <= 1.0e-7); + REQUIRE(ratios[2] == 6.0f); + REQUIRE(ratios[3] == 5.0f); + if(calculateSphericity) + { + const auto sphericities = ReadSurfaceMetricArray(dataStructure, k_MetricSphericityPath, {4}, scope); + REQUIRE(sphericities.size() == 4); + REQUIRE(sphericities[0] == 0.0f); + const std::array volumes{0.0, 122.0, 1.0, 2.0}; + const std::array areas{0.0, 16.0, 6.0, 10.0}; + for(usize featureIdx = 1; featureIdx < volumes.size(); ++featureIdx) + { + CAPTURE(featureIdx); + // The sphere of equal volume has area cbrt(36*pi*V*V). + const float64 expectedSphericity = std::cbrt(36.0 * std::numbers::pi_v * volumes[featureIdx] * volumes[featureIdx]) / areas[featureIdx]; + // This tolerance permits float32 arithmetic and the algorithm's rounded fractional exponents. + REQUIRE(std::abs(static_cast(sphericities[featureIdx]) - expectedSphericity) <= 5.0e-5 * expectedSphericity); + } + } + else + { + REQUIRE(dataStructure.getData(k_MetricSphericityPath) == nullptr); + } + REQUIRE(ReadSurfaceMetricArray(dataStructure, k_MetricIdsPath, {5, 5, 5}, scope) == featureIds); + REQUIRE(ReadSurfaceMetricArray(dataStructure, k_MetricCountsPath, {4}, scope) == counts); + RequireSurfaceMetricGeometry(dataStructure, *imageGeom, spacing, counts.size()); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} diff --git a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp index c3f5365574..effa28cc57 100644 --- a/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp +++ b/src/Plugins/SimplnxCore/test/DREAM3DFileTest.cpp @@ -4,6 +4,7 @@ #include "SimplnxCore/Filters/WriteDREAM3DFilter.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" +#include "simplnx/Common/ScopeGuard.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" @@ -11,6 +12,7 @@ #include "simplnx/DataStructure/DataGroup.hpp" #include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/EmptyListStore.hpp" #include "simplnx/DataStructure/Geometry/EdgeGeom.hpp" #include "simplnx/DataStructure/Geometry/HexahedralGeom.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" @@ -21,10 +23,15 @@ #include "simplnx/DataStructure/Geometry/VertexGeom.hpp" #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/DataStructure/IDataStore.hpp" +#include "simplnx/DataStructure/IO/Generic/IDataStoreFormatResolver.hpp" +#include "simplnx/DataStructure/IO/Generic/IOConstants.hpp" +#include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" #include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" #include "simplnx/DataStructure/ListStore.hpp" #include "simplnx/DataStructure/Montage/GridMontage.hpp" +#include "simplnx/DataStructure/NeighborList.hpp" #include "simplnx/DataStructure/ScalarData.hpp" +#include "simplnx/DataStructure/StringArray.hpp" #include "simplnx/Filter/Arguments.hpp" #include "simplnx/Filter/FilterHandle.hpp" #include "simplnx/Parameters/Dream3dImportParameter.hpp" @@ -33,6 +40,7 @@ #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" #include "simplnx/UnitTest/HDF5DatasetProbe.hpp" +#include "simplnx/UnitTest/ScopedDefaultFormatResolver.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/Utilities/ArrayCreationUtilities.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" @@ -43,10 +51,13 @@ #include +#include #include #include #include +#include #include +#include #include #include #include @@ -2121,3 +2132,323 @@ TEST_CASE("DREAM3DFileTest: PreflightCache avoids re-reading unchanged files", " REQUIRE(pipeline.preflight()); REQUIRE(cache.missCount() == missesBeforeRewritePreflight + 1); } + +namespace +{ +class C8FilterResolver : public IDataStoreFormatResolver +{ +public: + C8FilterResolver() = default; + ~C8FilterResolver() override = default; + C8FilterResolver(const C8FilterResolver&) = default; + C8FilterResolver(C8FilterResolver&&) = default; + C8FilterResolver& operator=(const C8FilterResolver&) = default; + C8FilterResolver& operator=(C8FilterResolver&&) = default; + + bool reject = false; + bool requireContext = false; + mutable std::vector paths; + + std::string resolveFormat(const DataStructure& dataStructure, const DataPath& path, DataType, uint64) const override + { + paths.push_back(path); + if(reject) + { + throw std::runtime_error("injected C8 filter policy rejection at '" + path.toString() + "'"); + } + if(requireContext) + { + REQUIRE(dataStructure.containsData(DataPath({"DestinationMarker"}))); + REQUIRE(dataStructure.containsData(DataPath({"Image", "Cells", "Selected"}))); + REQUIRE(dataStructure.containsData(DataPath({"Image", "Lists"}))); + REQUIRE(dataStructure.containsData(DataPath({"Image", "Labels"}))); + REQUIRE_FALSE(dataStructure.containsData(DataPath({"Image", "Cells", "Excluded"}))); + } + return ""; + } +}; + +void WriteC8FilterSelectionFile(const fs::path& path) +{ + DataStructure source; + source.setFormatResolver(std::make_shared()); + auto* imageGeom = ImageGeom::Create(source, "Image"); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({2, 2, 1}); + imageGeom->setSpacing({1.0F, 2.0F, 3.0F}); + imageGeom->setOrigin({-2.0F, -1.0F, 0.0F}); + auto* cells = AttributeMatrix::Create(source, "Cells", ShapeType{1, 2, 2}, imageGeom->getId()); + REQUIRE(cells != nullptr); + imageGeom->setCellData(*cells); + auto selectedStore = DataStoreUtilities::CreateDataStore(source, DataPath({"Image", "Cells", "Selected"}), {1, 2, 2}, {1}); + REQUIRE(selectedStore != nullptr); + const std::vector values{4, -7, 12, 29}; + REQUIRE(selectedStore->copyFromBuffer(0, nonstd::span(values.data(), values.size())).valid()); + REQUIRE(Int32Array::Create(source, "Selected", selectedStore, cells->getId()) != nullptr); + auto excludedStore = DataStoreUtilities::CreateDataStore(source, DataPath({"Image", "Cells", "Excluded"}), {1, 2, 2}, {1}); + REQUIRE(excludedStore != nullptr); + excludedStore->fill(90); + REQUIRE(Int32Array::Create(source, "Excluded", excludedStore, cells->getId()) != nullptr); + auto* lists = NeighborList::Create(source, "Lists", ShapeType{4}, imageGeom->getId()); + REQUIRE(lists != nullptr); + const std::vector> listValues{{1, 2}, {}, {-3}, {4, 5, 6}}; + for(int32 tupleIdx = 0; tupleIdx < 4; ++tupleIdx) + { + lists->setList(tupleIdx, listValues[static_cast(tupleIdx)]); + } + REQUIRE(StringArray::CreateWithValues(source, "Labels", ShapeType{2}, {"first", "second"}, imageGeom->getId()) != nullptr); + auto write = DREAM3D::WriteFile(path, source); + SIMPLNX_RESULT_REQUIRE_VALID(write); + fs::last_write_time(path, fs::last_write_time(path) - std::chrono::seconds(10)); +} + +void WriteC8FilterLegacyWarningFile(const fs::path& path) +{ + { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + REQUIRE(file.writeStringAttribute("FileVersion", DREAM3D::k_LegacyFileVersion.str()).valid()); + auto containers = file.createGroup("DataContainers"); + auto container = containers.createGroup("StatsContainer"); + auto attributeMatrix = container.createGroup("StatsAM"); + REQUIRE(attributeMatrix.writeVectorAttribute("TupleDimensions", ShapeType{1}).valid()); + REQUIRE(attributeMatrix.writeScalarAttribute("AttributeMatrixType", 13).valid()); + auto statistics = attributeMatrix.createGroup("StatisticsSource"); + REQUIRE(statistics.writeStringAttribute(nx::core::Constants::k_ObjectTypeTag, "Statistics").valid()); + auto values = statistics.createDataset("Values"); + const std::vector payload{1.0F}; + REQUIRE(values.writeSpan({1, 1}, nonstd::span(payload.data(), payload.size())).valid()); + REQUIRE(values.writeStringAttribute(nx::core::Constants::k_ObjectTypeTag, DataArray::GetTypeName()).valid()); + REQUIRE(values.writeVectorAttribute("TupleDimensions", ShapeType{1}).valid()); + REQUIRE(values.writeVectorAttribute("ComponentDimensions", ShapeType{1}).valid()); + REQUIRE(values.writeVectorAttribute("Mean", {4.0F, 5.0F, 6.0F}).valid()); + auto unsupported = attributeMatrix.createGroup("Unsupported"); + REQUIRE(unsupported.writeStringAttribute(nx::core::Constants::k_ObjectTypeTag, "UnsupportedLegacyObject").valid()); + } + fs::last_write_time(path, fs::last_write_time(path) - std::chrono::seconds(10)); +} +} // namespace + +TEST_CASE("Scoped default format resolver preserves ownership and nested restoration", "[Core][FormatResolver]") +{ + UnitTest::LoadPlugins(); + const auto incomingResolver = DataStructure::getDefaultFormatResolver(); + const bool throwFromNestedScope = GENERATE(false, true); + CAPTURE(throwFromNestedScope); + auto outerResolver = std::make_shared(); + const auto* outerResolverPtr = outerResolver.get(); + const std::weak_ptr outerLifetime = outerResolver; + { + const auto restoreIncoming = UnitTest::SetDefaultFormatResolverForScope(std::move(outerResolver)); + REQUIRE(outerResolver == nullptr); + CHECK(&DataStructure{}.formatResolver() == outerResolverPtr); + bool caughtException = false; + try + { + auto innerResolver = std::make_shared(); + const auto* innerResolverPtr = innerResolver.get(); + const auto restoreOuter = UnitTest::SetDefaultFormatResolverForScope(std::move(innerResolver)); + REQUIRE(innerResolver == nullptr); + REQUIRE_FALSE(outerLifetime.expired()); + CHECK(&DataStructure{}.formatResolver() == innerResolverPtr); + if(throwFromNestedScope) + { + throw std::runtime_error("resolver scope unwind"); + } + } catch(const std::runtime_error&) + { + caughtException = true; + } + CHECK(caughtException == throwFromNestedScope); + REQUIRE_FALSE(outerLifetime.expired()); + CHECK(&DataStructure{}.formatResolver() == outerResolverPtr); + } + CHECK(outerLifetime.expired()); + CHECK(&DataStructure{}.formatResolver() == incomingResolver.get()); + const auto restoredResolver = DataStructure::getDefaultFormatResolver(); + CHECK_FALSE(restoredResolver.owner_before(incomingResolver)); + CHECK_FALSE(incomingResolver.owner_before(restoredResolver)); +} + +TEST_CASE("Scoped null default format resolver preserves the active policy", "[Core][FormatResolver]") +{ + UnitTest::LoadPlugins(); + const auto incomingResolver = DataStructure::getDefaultFormatResolver(); + { + const auto restoreResolver = UnitTest::SetDefaultFormatResolverForScope(nullptr); + CHECK(&DataStructure{}.formatResolver() == incomingResolver.get()); + } + CHECK(&DataStructure{}.formatResolver() == incomingResolver.get()); +} + +TEST_CASE("C8 ReadDREAM3DFilter plans selected data in the destination context", "[C8][ReadDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + const auto policy = GENERATE(Dream3dImportParameter::PathImportPolicy::IncludeList, Dream3dImportParameter::PathImportPolicy::ExcludeList); + const bool warmCache = GENERATE(false, true); + const bool execute = GENERATE(false, true); + const bool rejectDestination = GENERATE(false, true); + CAPTURE(policy, warmCache, execute, rejectDestination); + const auto path = GetTestFilePath("c8_filter_destination_context.dream3d"); + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + cache.clear(); + cache.resetStats(); + const auto cleanup = MakeScopeGuard([&cache, &path]() noexcept { + cache.clear(); + std::error_code error; + fs::remove(path, error); + }); + WriteC8FilterSelectionFile(path); + if(warmCache) + { + auto warm = cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(warm); + } + auto processResolver = std::make_shared(); + processResolver->reject = true; + const auto restoreResolver = UnitTest::SetDefaultFormatResolverForScope(processResolver); + auto destinationResolver = std::make_shared(); + destinationResolver->requireContext = true; + destinationResolver->reject = rejectDestination; + DataStructure destination; + destination.setFormatResolver(destinationResolver); + // The marker uses an explicit resident store because the process policy deliberately rejects every request. + auto* marker = Int32Array::Create(destination, "DestinationMarker", std::make_shared>(ShapeType{1}, ShapeType{1}, 73)); + REQUIRE(marker != nullptr); + const auto markerId = marker->getId(); + const auto* originalStore = marker->getIDataStore(); + const auto originalPaths = destination.getAllDataPaths(); + const DataPath selectedPath({"Image", "Cells", "Selected"}); + const DataPath listsPath({"Image", "Lists"}); + const DataPath labelsPath({"Image", "Labels"}); + const std::vector paths = + policy == Dream3dImportParameter::PathImportPolicy::IncludeList ? std::vector{selectedPath, listsPath, labelsPath} : std::vector{DataPath({"Image", "Cells", "Excluded"})}; + ReadDREAM3DFilter filter; + Arguments args; + args.insertOrAssign(ReadDREAM3DFilter::k_ImportFileData, Dream3dImportParameter::ImportData(path, policy, paths)); + cache.resetStats(); + Result<> result; + if(execute) + { + result = filter.execute(destination, args).result; + } + else + { + auto preflight = filter.preflight(destination, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + CHECK(destination.getAllDataPaths() == originalPaths); + CHECK(destinationResolver->paths.empty()); + CHECK(processResolver->paths.empty()); + result = preflight.outputActions.value().applyRegular(destination, IDataAction::Mode::Preflight); + } + CHECK(cache.missCount() == (warmCache ? 0 : 1)); + CHECK(cache.hitCount() == (warmCache ? 2 : 1)); + CHECK(processResolver->paths.empty()); + CHECK(destination.getData(markerId) == marker); + CHECK(marker->getIDataStore() == originalStore); + CHECK((*marker)[0] == 73); + CHECK(std::count(destinationResolver->paths.begin(), destinationResolver->paths.end(), selectedPath) == 1); + CHECK(std::count(destinationResolver->paths.begin(), destinationResolver->paths.end(), DataPath({"Image", "Cells", "Excluded"})) == 0); + if(rejectDestination) + { + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().message.find("injected C8 filter policy rejection") != std::string::npos); + CHECK(result.errors().front().message.find(selectedPath.toString()) != std::string::npos); + CHECK(destination.getAllDataPaths() == originalPaths); + return; + } + SIMPLNX_RESULT_REQUIRE_VALID(result); + const Int32Array* selectedPtr = nullptr; + REQUIRE_NOTHROW(selectedPtr = &destination.getDataRefAs(selectedPath)); + const NeighborList* listsPtr = nullptr; + REQUIRE_NOTHROW(listsPtr = &destination.getDataRefAs>(listsPath)); + const StringArray* labelsPtr = nullptr; + REQUIRE_NOTHROW(labelsPtr = &destination.getDataRefAs(labelsPath)); + const ImageGeom* imageGeomPtr = nullptr; + REQUIRE_NOTHROW(imageGeomPtr = &destination.getDataRefAs(DataPath({"Image"}))); + const AttributeMatrix* cellsPtr = nullptr; + REQUIRE_NOTHROW(cellsPtr = &destination.getDataRefAs(DataPath({"Image", "Cells"}))); + CHECK(imageGeomPtr->getCellData() == cellsPtr); + CHECK(imageGeomPtr->getCellDataId() == cellsPtr->getId()); + CHECK(imageGeomPtr->getDimensions() == SizeVec3{2, 2, 1}); + CHECK(imageGeomPtr->getOrigin() == FloatVec3{-2.0F, -1.0F, 0.0F}); + CHECK(imageGeomPtr->getSpacing() == FloatVec3{1.0F, 2.0F, 3.0F}); + CHECK_FALSE(destination.containsData(DataPath({"Image", "Cells", "Excluded"}))); + if(execute) + { + CHECK(selectedPtr->getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(selectedPtr->size() == 4); + CHECK((*selectedPtr)[0] == 4); + CHECK((*selectedPtr)[1] == -7); + CHECK((*selectedPtr)[2] == 12); + CHECK((*selectedPtr)[3] == 29); + CHECK(listsPtr->getList(0) == std::vector{1, 2}); + CHECK(listsPtr->getList(1).empty()); + CHECK(listsPtr->getList(2) == std::vector{-3}); + CHECK(listsPtr->getList(3) == std::vector{4, 5, 6}); + REQUIRE_FALSE(labelsPtr->isPlaceholder()); + REQUIRE(labelsPtr->getNumberOfTuples() == 2); + CHECK((*labelsPtr)[0] == "first"); + CHECK((*labelsPtr)[1] == "second"); + } + else + { + REQUIRE(selectedPtr->getStoreType() == IDataStore::StoreType::Empty); + REQUIRE(dynamic_cast*>(listsPtr->getStore().get()) != nullptr); + CHECK(labelsPtr->isPlaceholder()); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("C8 ReadDREAM3DFilter retains metadata warnings on unsupported policy", "[C8][ReadDREAM3DFilter]") +{ + UnitTest::LoadPlugins(); + const bool warmCache = GENERATE(false, true); + const bool execute = GENERATE(false, true); + CAPTURE(warmCache, execute); + const auto path = GetTestFilePath("c8_filter_unsupported_policy.dream3d"); + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + cache.clear(); + const auto cleanup = MakeScopeGuard([&cache, &path]() noexcept { + cache.clear(); + std::error_code error; + fs::remove(path, error); + }); + WriteC8FilterLegacyWarningFile(path); + if(warmCache) + { + auto warm = cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(warm); + REQUIRE(warm.warnings().size() == 1); + } + DataStructure destination; + auto* marker = DataGroup::Create(destination, "DestinationMarker"); + REQUIRE(marker != nullptr); + const auto markerId = marker->getId(); + const auto originalPaths = destination.getAllDataPaths(); + ReadDREAM3DFilter filter; + Arguments args; + args.insertOrAssign(ReadDREAM3DFilter::k_ImportFileData, Dream3dImportParameter::ImportData(path, static_cast(99), {})); + cache.resetStats(); + Result<> result; + if(execute) + { + result = filter.execute(destination, args).result; + } + else + { + auto preflight = filter.preflight(destination, args); + result = ConvertResult(std::move(preflight.outputActions)); + } + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -51); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings().front().code == -298012); + CHECK(result.warnings().front().message == "DataObject 'Unsupported' is not a supported simplnx data type"); + CHECK(cache.missCount() == (warmCache ? 0 : 1)); + CHECK(cache.hitCount() == (warmCache ? 1 : 0)); + CHECK(destination.getData(markerId) == marker); + CHECK(destination.getAllDataPaths() == originalPaths); +} diff --git a/src/Plugins/SimplnxCore/test/FlyingEdges3DTest.cpp b/src/Plugins/SimplnxCore/test/FlyingEdges3DTest.cpp index a44b9e225f..032ee0c7eb 100644 --- a/src/Plugins/SimplnxCore/test/FlyingEdges3DTest.cpp +++ b/src/Plugins/SimplnxCore/test/FlyingEdges3DTest.cpp @@ -45,10 +45,13 @@ TEST_CASE("SimplnxCore::Image Contouring Valid Execution", "[SimplnxCore][Flying const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); CAPTURE(scenario); + UnitTest::AlgorithmTestScope algorithmTestScope(scenario); // Read Exemplar DREAM3D File Filter auto exemplarFilePath = fs::path(fmt::format("{}/flying_edges_exemplar.dream3d", unit_test::k_TestFilesDir)); DataStructure dataStructure = LoadDataStructure(exemplarFilePath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(ContourTest::k_DataPath)); + algorithmTestScope.requireExpectedStore(dataStructure.getDataRefAs(ContourTest::k_DataPath)); { // Instantiate the filter and an Arguments Object @@ -67,35 +70,41 @@ TEST_CASE("SimplnxCore::Image Contouring Valid Execution", "[SimplnxCore][Flying auto preflightResult = filter.preflight(dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); - // This is in here because the exemplar face attribute matrix is not sized correctly. This will - // correct that value allowing the test to proceed normally. + // The archive has 70 faces but an incorrect face AttributeMatrix size. + REQUIRE_NOTHROW(dataStructure.getDataRefAs(ContourTest::k_ExemplarContourPath)); auto& exemplarContourTriGeom = dataStructure.getDataRefAs(ContourTest::k_ExemplarContourPath); auto faceAttributeMatrixResizeResult = exemplarContourTriGeom.getFaceAttributeMatrixRef().resizeTuples({70}); SIMPLNX_RESULT_REQUIRE_VALID(faceAttributeMatrixResizeResult); // Execute the filter and check the result - UnitTest::AlgorithmTestScope algorithmTestScope(scenario); auto executeResult = algorithmTestScope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); } // Compare the results { + REQUIRE_NOTHROW(dataStructure.getDataRefAs(ContourTest::k_NewContourPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(ContourTest::k_ExemplarContourPath)); auto& newContourTriGeom = dataStructure.getDataRefAs(ContourTest::k_NewContourPath); auto& exemplarContourTriGeom = dataStructure.getDataRefAs(ContourTest::k_ExemplarContourPath); const auto& kNxVertArray = newContourTriGeom.getVerticesRef(); const auto& kExemplarsVertArray = exemplarContourTriGeom.getVerticesRef(); + algorithmTestScope.requireExpectedStore(kNxVertArray); CompareDataArrays(kExemplarsVertArray, kNxVertArray); const auto& kNxTriArray = newContourTriGeom.getFacesRef(); const auto& kExemplarsTriArray = exemplarContourTriGeom.getFacesRef(); + algorithmTestScope.requireExpectedStore(kNxTriArray); CompareDataArrays(kExemplarsTriArray, kNxTriArray); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(ContourTest::k_NewNormals)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(ContourTest::k_ExemplarNormals)); const auto& kNxNormalsArray = dataStructure.getDataRefAs(ContourTest::k_NewNormals); const auto& kExemplarsNormalsArray = dataStructure.getDataRefAs(ContourTest::k_ExemplarNormals); + algorithmTestScope.requireExpectedStore(kNxNormalsArray); CompareDataArrays(kExemplarsNormalsArray, kNxNormalsArray); } diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleBatchControlUtilities.hpp b/src/Plugins/SimplnxCore/test/IdentifySampleBatchControlUtilities.hpp new file mode 100644 index 0000000000..c84e2b67e7 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/IdentifySampleBatchControlUtilities.hpp @@ -0,0 +1,138 @@ +#pragma once + +#include "IdentifySampleBenchmarkUtilities.hpp" +#include "simplnx/Common/ScopeGuard.hpp" + +#include +#include + +/** + * @namespace IdentifySampleBatchControlTest + * @brief Shares exact reservation and output checks between resident and actual-OOC fixtures. + */ +namespace IdentifySampleBatchControlTest +{ +using namespace nx::core; +using namespace IdentifySampleBenchmarkTest; + +/** + * @brief Runs an actual filter with a real competing reservation and checks every mask value. + * @tparam T Bool or UInt8 mask element type. + * @tparam ScopeT Resident or private actual-OOC dispatch witness. + * @param scope Owns the caller-selected scenario. + * @param data Initialized mask; fixture creation occurs before budget occupation. + * @param width Requested stable batch width from one through eight. + * @param partialBytes Extra headroom below one whole plane. + * @param fillHoles Selects the independent filled-hole oracle. + * @param fixedCount Number of YZ planes. + * @param expectCcl True when the selected dispatch path must use CCL. + */ +template +void CheckGrant(ScopeT& scope, DataStructure& data, usize width, usize partialBytes, bool fillHoles, usize fixedCount = 19, bool expectCcl = true) +{ + REQUIRE(width >= 1); + REQUIRE(width <= 8); + constexpr uint64 k_PlaneBytes = 11 * 13 * sizeof(T); + REQUIRE(partialBytes < k_PlaneBytes); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + auto* originalStore = mask.getDataStore(); + scope.requireExpectedStore(mask); + const auto originalId = mask.getId(); + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const uint64 originalBudget = budget.budgetBytes(); + LogicalCounters counters; + counters.selected = &mask; + { + const BudgetRestore restoreBudget; + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + const uint64 capacity = budget.maximumWorkingMemoryBytes(); + const uint64 headroom = (width - 1) * k_PlaneBytes + partialBytes; + auto occupied = budget.reserveWorkingMemory(capacity - headroom); + REQUIRE(occupied.sizeBytes() == capacity - headroom); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counters, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + REQUIRE(prior.callback == nullptr); + IdentifySampleFilter filter; + const auto args = IdentifySampleBatchTest::ArgumentsFor(2, fillHoles); + const auto result = scope.executeFilter(filter, data, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == occupied.sizeBytes()); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + REQUIRE(budget.budgetBytes() == originalBudget); + REQUIRE(mask.getDataStore() == originalStore); + REQUIRE(mask.getId() == originalId); + scope.requireExpectedStore(mask); + REQUIRE_FALSE(counters.invalid); + REQUIRE_FALSE(counters.invalidExtraAccounting); + REQUIRE(counters.active == 0); + REQUIRE(counters.extraLiveBytes == 0); + REQUIRE(counters.extraRetainedBytes == 0); + REQUIRE(counters.planeLive == 0); + REQUIRE(counters.xyLive == 0); + if(expectCcl) + { + // Replay the admitted schedule independently. An out-of-core mask publishes its first batch without extra planes. + const bool firstBatchSingle = mask.getDataStoreRef().getStoreType() == IDataStore::StoreType::OutOfCore; + const uint64 headroom = (width - 1) * k_PlaneBytes + partialBytes; + usize batches = 0; + usize owningBatches = 0; + usize minimumWidth = std::numeric_limits::max(); + usize maximumWidth = 0; + usize allocatedPlanes = 0; + uint64 requestedMaximum = 0; + uint64 grantedMaximum = 0; + for(usize start = 0; start < fixedCount; ++batches) + { + const usize maxExtra = firstBatchSingle && batches == 0 ? 0 : std::min(usize{7}, fixedCount - start - 1); + const uint64 requested = maxExtra * k_PlaneBytes; + const uint64 granted = std::min(requested, headroom); + const usize extras = std::min(maxExtra, static_cast(granted / k_PlaneBytes)); + requestedMaximum = std::max(requestedMaximum, requested); + grantedMaximum = std::max(grantedMaximum, granted); + owningBatches += static_cast(extras != 0); + allocatedPlanes += extras; + minimumWidth = std::min(minimumWidth, extras + 1); + maximumWidth = std::max(maximumWidth, extras + 1); + start += extras + 1; + } + REQUIRE(counters.batchCount == batches); + REQUIRE(counters.batchPlanes == fixedCount); + REQUIRE(counters.minimumBatchWidth == minimumWidth); + REQUIRE(counters.maximumBatchWidth == maximumWidth); + REQUIRE(counters.extraRequestCount == batches); + REQUIRE(counters.extraGrantCount == batches); + REQUIRE(counters.extraRetainCount == batches); + REQUIRE(counters.extraReservationReleaseCount == batches); + REQUIRE(counters.extraAllocationCount == owningBatches); + REQUIRE(counters.extraReleaseCount == owningBatches); + REQUIRE(counters.extraAllocatedBytes == allocatedPlanes * k_PlaneBytes); + REQUIRE(counters.extraPeakBytes == (maximumWidth - 1) * k_PlaneBytes); + REQUIRE(counters.extraRetainedMaxBytes == counters.extraPeakBytes); + REQUIRE(counters.extraRequestedMaxBytes == requestedMaximum); + REQUIRE(counters.extraGrantedMaxBytes == grantedMaximum); + REQUIRE(counters.reads == 2 * batches * 13); + REQUIRE(counters.writes == batches * 13); + REQUIRE(counters.readSuccess == counters.reads); + REQUIRE(counters.writeSuccess == counters.writes); + REQUIRE(counters.maximumSpan == fixedCount * 11); + REQUIRE(counters.concurrent == 1); + REQUIRE(counters.allocationEvents == 2); + REQUIRE(counters.releaseEvents == 2); + REQUIRE(counters.namedCarrierPeak == (11 * 13 + fixedCount * 11) * sizeof(T)); + } + else + { + // The resident BFS route does not own CCL slice carriers or reservations. + REQUIRE(counters.reads == 0); + REQUIRE(counters.writes == 0); + REQUIRE(counters.allocationEvents == 0); + REQUIRE(counters.extraAllocationCount == 0); + } + IdentifySampleBatchTest::RequireOutput(data, 2, fillHoles, fixedCount); +} +} // namespace IdentifySampleBatchControlTest diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleBatchFailureTest.cpp b/src/Plugins/SimplnxCore/test/IdentifySampleBatchFailureTest.cpp new file mode 100644 index 0000000000..dc947ce1de --- /dev/null +++ b/src/Plugins/SimplnxCore/test/IdentifySampleBatchFailureTest.cpp @@ -0,0 +1,2089 @@ +#include "IdentifySampleBenchmarkUtilities.hpp" +#include "SimplnxCore/Filters/Algorithms/IdentifySample.hpp" + +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/IO/Generic/DataIOCollection.hpp" +#include "simplnx/DataStructure/IO/Generic/IDataIOManager.hpp" +#include "simplnx/Pipeline/Pipeline.hpp" +#include "simplnx/Pipeline/PipelineFilter.hpp" +#include "simplnx/Utilities/InMemoryTemporaryRecordStore.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +using namespace IdentifySampleBenchmarkTest; +constexpr usize k_X = 19; +constexpr usize k_Y = 11; +constexpr usize k_Z = 13; +constexpr int32 k_PrimaryCode = -77891; +constexpr int32 k_SecondaryCode = -77892; +constexpr int32 k_PrimaryWarning = -77893; +constexpr int32 k_SecondaryWarning = -77894; + +/** + * @enum FaultKind + * @brief Selects a carrier without changing the actual failing operation. + */ +enum class FaultKind +{ + None, ///< Leaves the selected operation unchanged. + Result, ///< Returns a distinct error and warning. + Standard, ///< Throws PlaneFailure with its original text and payload. + Allocation, ///< Throws AllocationFailure with its original payload. + Nonstandard ///< Throws a payload that does not derive from std::exception. +}; +/** + * @enum CopyPhase + * @brief Identifies actual mask-copy phases in the first batch. + */ +enum class CopyPhase +{ + GatherRead, ///< Reads source values before any plane finishes. + CommitRead, ///< Reads fresh source values before completed planes are written. + CommitWrite ///< Writes source values with only the completed planes replaced. +}; +/** + * @enum CancelAt + * @brief Selects independently witnessed cancellation sites. + */ +enum class CancelAt +{ + None, ///< Leaves the cancellation flag clear. + Pre, ///< Sets cancellation before the target call. + Gather, ///< Sets cancellation in the Z=1 source read. + BeforePlane, ///< Sets cancellation in the selected plane's progress message. + RecordWrite, ///< Sets cancellation after a successful scratch write inside CCL. + CommitRead, ///< Sets cancellation in the first completed-prefix read. + WithPrimary, ///< Sets cancellation when the record read also fails. + WithSecondary ///< Sets cancellation when publication also fails. +}; +/** + * @enum HandlerKind + * @brief Distinguishes valid empty callbacks from Error-only throwing callbacks. + */ +enum class HandlerKind +{ + Empty, ///< Requests an empty handler unless a progress control is armed. + Capturing, ///< Counts ordinary messages without adding an Error failure. + ErrorThrow ///< Throws from the Error-message branch. +}; + +/** + * @class PlaneFailure + * @brief Carries a test payload that must survive a lone standard exception. + */ +class PlaneFailure : public std::runtime_error +{ +public: + /** + * @brief Stores the payload and chooses the primary or secondary error text. + * @param value Exception payload; 101 selects the primary message. + */ + explicit PlaneFailure(int value) + : std::runtime_error(value == 101 ? "primary record payload 101" : "secondary copy payload 202") + , payload(value) + { + } + + ~PlaneFailure() override = default; + PlaneFailure(const PlaneFailure&) = default; + PlaneFailure(PlaneFailure&&) = default; + PlaneFailure& operator=(const PlaneFailure&) = default; + PlaneFailure& operator=(PlaneFailure&&) = default; + + int payload; +}; +/** + * @class AllocationFailure + * @brief Proves that allocation subclasses retain their payload on a lone failure. + */ +class AllocationFailure : public std::bad_alloc +{ +public: + /** + * @brief Stores a payload that identifies the original allocation failure. + * @param value Payload checked after the exception leaves the target call. + */ + explicit AllocationFailure(int value) + : payload(value) + { + } + + ~AllocationFailure() override = default; + AllocationFailure(const AllocationFailure&) = default; + AllocationFailure(AllocationFailure&&) = default; + AllocationFailure& operator=(const AllocationFailure&) = default; + AllocationFailure& operator=(AllocationFailure&&) = default; + + int payload; +}; +/** + * @struct NonstandardFailure + * @brief Supplies a nonstandard exception with an independently checked payload. + */ +struct NonstandardFailure +{ + int payload; +}; + +/** + * @brief Produces the selected error or exception at a witnessed test operation. + * @param kind Chooses the Result or exception carrier. + * @param primary True for the plane failure, false for its publication failure. + * @return A distinct error and warning for a Result failure, otherwise a valid Result. + * @throws PlaneFailure When kind selects a standard exception. + * @throws AllocationFailure When kind selects an allocation exception. + * @throws NonstandardFailure When kind selects a nonstandard exception. + */ +Result<> RaiseFault(FaultKind kind, bool primary) +{ + const int payload = primary ? 101 : 202; + switch(kind) + { + case FaultKind::Result: { + auto result = MakeErrorResult(primary ? k_PrimaryCode : k_SecondaryCode, primary ? "primary record Result" : "secondary copy Result"); + result.warnings().push_back({primary ? k_PrimaryWarning : k_SecondaryWarning, primary ? "primary warning" : "secondary warning"}); + return result; + } + case FaultKind::Standard: + throw PlaneFailure(payload); + case FaultKind::Allocation: + throw AllocationFailure(payload); + case FaultKind::Nonstandard: + throw NonstandardFailure{payload}; + case FaultKind::None: + return {}; + } + return {}; +} + +/** + * @struct FailureState + * @brief Collects actual record/copy witnesses; all fixtures execute serially. + * @note A null selected array tests identity filtering. State and any selected array outlive all installed callbacks and decorated stores. + */ +struct FailureState +{ + FaultKind primary = FaultKind::None; + FaultKind secondary = FaultKind::None; + CopyPhase secondaryPhase = CopyPhase::CommitWrite; + CancelAt cancelAt = CancelAt::None; + usize primaryPlane = 1; + usize secondaryZ = 1; + bool messagePrimary = false; + bool aggregateFailure = false; + bool diagnosticMatch = true; + bool armed = false; + bool primaryFired = false; + bool secondaryFired = false; + bool cancelFired = false; + usize aggregateCalls = 0; + usize recordFactories = 0; + usize liveRecords = 0; + usize recordReads = 0; + usize recordWrites = 0; + usize reads = 0; + usize writes = 0; + usize successfulWrites = 0; + usize progressCalls = 0; + usize errorCalls = 0; + const IDataArray* selected = nullptr; + std::atomic_bool cancel = false; +}; + +/** + * @class RecordDecorator + * @brief Injects only from an actual CCL scratch read or successful scratch write. + */ +class RecordDecorator : public ITemporaryRecordStore +{ +public: + /** + * @brief Takes ownership of the real scratch store and records its plane identity. + * @param delegate Owns the validated resident record store. + * @param state Receives operation counts and supplies faults; outlives this decorator. + * @param plane Zero-based plane selected by actual record-provider creation order. + */ + RecordDecorator(std::unique_ptr delegate, FailureState& state, usize plane) + : m_Delegate(std::move(delegate)) + , m_State(state) + , m_Plane(plane) + { + ++m_State.liveRecords; + } + + RecordDecorator(const RecordDecorator&) = delete; + RecordDecorator(RecordDecorator&&) = delete; + RecordDecorator& operator=(const RecordDecorator&) = delete; + RecordDecorator& operator=(RecordDecorator&&) = delete; + + /** + * @brief Marks the decorator as closed before its owned store is destroyed. + */ + ~RecordDecorator() noexcept override + { + --m_State.liveRecords; + } + uint64 recordSize() const override + { + return m_Delegate->recordSize(); + } + uint64 recordCount() const override + { + return m_Delegate->recordCount(); + } + uint64 maxRecordsPerBatch() const override + { + return m_Delegate->maxRecordsPerBatch(); + } + bool isReadOnly() const override + { + return m_Delegate->isReadOnly(); + } + + /** + * @brief Delegates one CCL read or injects the selected primary carrier. + * @param offset First scratch record. + * @param count Number of requested records. + * @param records Receives the record bytes. + * @param cancel Cancellation flag passed to the real store. + * @return Actual read result or the deliberate primary Result. + * @throws PlaneFailure For the selected standard primary fault. + * @throws AllocationFailure For the selected allocation primary fault. + * @throws NonstandardFailure For the selected nonstandard primary fault. + */ + Result read(uint64 offset, uint64 count, nonstd::span records, const std::atomic_bool& cancel) const override + { + ++m_State.recordReads; + if(!m_State.messagePrimary && m_Plane == m_State.primaryPlane && m_State.primary != FaultKind::None && !m_State.primaryFired) + { + m_State.primaryFired = true; + if(m_State.cancelAt == CancelAt::WithPrimary) + { + m_State.cancel = true; + m_State.cancelFired = true; + } + return ConvertResultTo(RaiseFault(m_State.primary, true), 0); + } + return m_Delegate->read(offset, count, records, cancel); + } + + /** + * @brief Delegates a scratch write before setting the selected mid-CCL cancellation. + * @param offset First scratch record. + * @param count Number of records. + * @param records Supplies the record bytes. + * @param cancel Cancellation flag passed to the real store. + * @return The real store's Result. + */ + Result<> write(uint64 offset, uint64 count, nonstd::span records, const std::atomic_bool& cancel) override + { + ++m_State.recordWrites; + auto result = m_Delegate->write(offset, count, records, cancel); + if(result.valid() && m_Plane == m_State.primaryPlane && m_State.cancelAt == CancelAt::RecordWrite) + { + // Cancel after a real flush, while the unchanged per-plane CCL is still active. + m_State.cancel = true; + m_State.cancelFired = true; + } + return result; + } + /** + * @brief Passes a fill to the real record store without adding a fault. + * @param offset First record to fill. + * @param count Number of records to fill. + * @param record Supplies one complete record to repeat. + * @param cancel Cancellation flag passed to the real store. + * @return The real store's fill Result. + */ + Result<> fill(uint64 offset, uint64 count, nonstd::span record, const std::atomic_bool& cancel) override + { + return m_Delegate->fill(offset, count, record, cancel); + } + /** + * @brief Passes a size change to the real record store without adding a fault. + * @param count New logical record count. + * @param cancel Cancellation flag passed to the real store. + * @return The real store's resize Result. + */ + Result<> resize(uint64 count, const std::atomic_bool& cancel) override + { + return m_Delegate->resize(count, cancel); + } + +private: + std::unique_ptr m_Delegate; + FailureState& m_State; + usize m_Plane; +}; + +/** + * @class RecordManager + * @brief Supplies real resident records behind a fault decorator for this one execution. + */ +class RecordManager : public IDataIOManager +{ +public: + /** + * @brief Borrows the execution state used by all created record decorators. + * @param state Receives provider counts and supplies faults; outlives this manager and its stores. + */ + explicit RecordManager(FailureState& state) + : m_State(state) + { + } + + ~RecordManager() override = default; + RecordManager(const RecordManager&) = default; + RecordManager(RecordManager&&) = default; + RecordManager& operator=(const RecordManager&) = delete; + RecordManager& operator=(RecordManager&&) = delete; + + std::string formatName() const override + { + return "IdentifySample test records"; + } + bool supportsTemporaryRecordStore() const override + { + return true; + } + /** + * @brief Creates the actual scratch owner, then assigns this plane's read/write decorator. + * @param config Production CCL record configuration. + * @return Decorated scratch store or its original creation error. + */ + Result> createTemporaryRecordStore(const TemporaryRecordStoreConfig& config) const override + { + auto result = InMemoryTemporaryRecordStore::Create(config); + if(result.invalid()) + { + return ConvertInvalidResult>(std::move(result)); + } + // With hole filling disabled, each plane creates exactly one equivalence store. + const usize plane = m_State.recordFactories++; + return {std::unique_ptr(new RecordDecorator(std::move(result.value()), m_State, plane))}; + } + +private: + FailureState& m_State; +}; + +/** + * @class ScopedRecordManager + * @brief Restores the exact first map entry after a serial, fixture-only execution. + * @note Construct all fixtures and filter/pipeline objects before this scope. No unrelated factories may run inside it. + */ +class ScopedRecordManager +{ +public: + /** + * @brief Replaces the first existing manager entry during one serial execution. + * @param state Receives provider calls and supplies record faults; outlives this scope. + * @pre The manager collection is nonempty and no concurrent factory activity occurs. + */ + explicit ScopedRecordManager(FailureState& state) + : m_Collection(DataStoreUtilities::GetIOCollection()) + , m_Entry(m_Collection.begin()) + , m_Original(m_Entry->second) + , m_Replacement(std::make_shared(state)) + { + m_Entry->second = m_Replacement; + } + /** + * @brief Restores the exact saved manager shared pointer. + */ + ~ScopedRecordManager() noexcept + { + m_Entry->second = m_Original; + } + ScopedRecordManager(const ScopedRecordManager&) = delete; + ScopedRecordManager(ScopedRecordManager&&) = delete; + ScopedRecordManager& operator=(const ScopedRecordManager&) = delete; + ScopedRecordManager& operator=(ScopedRecordManager&&) = delete; + /** + * @brief Checks that capability selection starts with this scope's record manager. + * @return True when the first collection entry still owns the installed manager. + */ + bool selected() const + { + return m_Collection.begin()->second == m_Replacement; + } + +private: + DataIOCollection& m_Collection; + DataIOCollection::iterator m_Entry; + std::shared_ptr m_Original; + std::shared_ptr m_Replacement; +}; + +/** + * @class MaskStore + * @brief Adds phase-aware faults to real resident mask copies after setup. + * @tparam T Bool or UInt8 mask element type. + */ +template +class MaskStore : public DataStore +{ +public: + /** + * @brief Creates the small resident mask and borrows its transfer/fault state. + * @param state Supplies faults only after setup arms the store; outlives this store. + */ + explicit MaskStore(FailureState& state) + : DataStore(ShapeType{k_Z, k_Y, k_X}, ShapeType{1}, std::optional{}) + , m_State(state) + { + } + + ~MaskStore() override = default; + MaskStore(const MaskStore&) = default; + MaskStore(MaskStore&&) = default; + MaskStore& operator=(const MaskStore&) = delete; + MaskStore& operator=(MaskStore&&) = delete; + + /** + * @brief Reads the mask or injects a witnessed gather/prefix-read fault. + * @param start First flat mask value. + * @param values Receives the requested values. + * @return Real read Result or deliberate publication Result. + * @throws PlaneFailure For the selected standard copy fault. + * @throws AllocationFailure For the selected allocation copy fault. + * @throws NonstandardFailure For the selected nonstandard copy fault. + */ + Result<> copyIntoBuffer(usize start, nonstd::span values) const override + { + if(m_State.armed) + { + ++m_State.reads; + const usize z = start / (k_X * k_Y); + const bool gather = m_State.recordFactories == 0 && !m_State.primaryFired; + const CopyPhase phase = gather ? CopyPhase::GatherRead : CopyPhase::CommitRead; + if((gather && m_State.cancelAt == CancelAt::Gather && z == 1) || (!gather && m_State.cancelAt == CancelAt::CommitRead && z == 0)) + { + m_State.cancel = true; + m_State.cancelFired = true; + } + if(m_State.secondary != FaultKind::None && phase == m_State.secondaryPhase && z == m_State.secondaryZ && !m_State.secondaryFired) + { + m_State.secondaryFired = true; + if(m_State.cancelAt == CancelAt::WithSecondary) + { + m_State.cancel = true; + m_State.cancelFired = true; + } + return RaiseFault(m_State.secondary, false); + } + } + return DataStore::copyIntoBuffer(start, values); + } + /** + * @brief Writes a mask prefix or fails before the selected Z write. + * @param start First flat mask value. + * @param values Supplies the replacement values. + * @return Real write Result or deliberate publication Result. + * @throws PlaneFailure For the selected standard copy fault. + * @throws AllocationFailure For the selected allocation copy fault. + * @throws NonstandardFailure For the selected nonstandard copy fault. + */ + Result<> copyFromBuffer(usize start, nonstd::span values) override + { + if(m_State.armed) + { + ++m_State.writes; + const usize z = start / (k_X * k_Y); + if(m_State.secondary != FaultKind::None && m_State.secondaryPhase == CopyPhase::CommitWrite && z == m_State.secondaryZ && !m_State.secondaryFired) + { + m_State.secondaryFired = true; + if(m_State.cancelAt == CancelAt::WithSecondary) + { + m_State.cancel = true; + m_State.cancelFired = true; + } + return RaiseFault(m_State.secondary, false); + } + } + auto result = DataStore::copyFromBuffer(start, values); + if(m_State.armed && result.valid()) + { + ++m_State.successfulWrites; + } + return result; + } + +private: + FailureState& m_State; +}; + +/** + * @brief Places the removable island at Z=0 so an early successful write is visible. + * @param x Selects the alternating rectangle position for this YZ plane. + * @param y First coordinate within the plane. + * @param z Second coordinate within the plane. + * @return True for the unfilled rectangle or the separate Z=0 island. + */ +bool FailureInput(usize x, usize y, usize z) +{ + return IdentifySampleBatchTest::ExpectedValue(y, z, x, false) || (y == 0 && z == 0); +} + +/** + * @brief Initializes the independent early-island fixture before faults are armed. + * @tparam T Bool or UInt8 mask type. + * @param store Reports real mask transfers during execution. + * @return Geometry and initialized mask. + */ +template +DataStructure CreateFailureFixture(const std::shared_ptr>& store) +{ + auto data = IdentifySampleBatchTest::CreateFixture(2, k_X, store); + std::array xy{}; + for(usize z = 0; z < k_Z; ++z) + { + for(usize y = 0; y < k_Y; ++y) + { + for(usize x = 0; x < k_X; ++x) + { + xy[y * k_X + x] = static_cast(FailureInput(x, y, z)); + } + } + const auto result = store->copyFromBuffer(z * xy.size(), nonstd::span(xy.data(), xy.size())); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + return data; +} + +/** + * @brief Compares all cells against only the completed X and successful Z prefixes. + * @tparam T Bool or UInt8 mask element type. + * @param data Contains the output mask after the fault controls are disarmed. + * @param completedPlanes Number of fully classified X planes, starting at zero. + * @param successfulZ Number of committed Z slices, starting at zero. + */ +template +void RequirePrefix(const DataStructure& data, usize completedPlanes, usize successfulZ) +{ + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + const auto& store = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath).getDataStoreRef(); + std::array xy{}; + for(usize z = 0; z < k_Z; ++z) + { + const auto result = store.copyIntoBuffer(z * xy.size(), nonstd::span(xy.data(), xy.size())); + SIMPLNX_RESULT_REQUIRE_VALID(result); + for(usize y = 0; y < k_Y; ++y) + { + for(usize x = 0; x < k_X; ++x) + { + const bool expected = x < completedPlanes && z < successfulZ ? IdentifySampleBatchTest::ExpectedValue(y, z, x, false) : FailureInput(x, y, z); + INFO("xyz=" << x << ',' << y << ',' << z << " completed=" << completedPlanes << " successful Z=" << successfulZ); + REQUIRE(xy[y * k_X + x] == static_cast(expected)); + } + } + } + UnitTest::CheckArraysInheritTupleDims(data); +} + +/** + * @brief Selects unfilled YZ classification for the terminal-failure fixtures. + * @return Slice-by-slice YZ inputs using the fixture geometry and mask paths, with hole filling disabled. + */ +IdentifySampleInputValues Inputs() +{ + IdentifySampleInputValues inputs{}; + inputs.FillHoles = false; + inputs.InputImageGeometryPath = IdentifySampleBatchTest::k_ImagePath; + inputs.MaskArrayPath = IdentifySampleBatchTest::k_MaskPath; + inputs.SliceBySlice = true; + inputs.SliceBySlicePlaneIndex = 2; + return inputs; +} + +/** + * @brief Builds an empty, observing, or deliberately failing message callback. + * @param state Records progress calls and the selected message/cancel event. + * @param handler Chooses how Error messages behave. + * @return Callback with a lifetime bounded by state. + */ +IFilter::MessageHandler Messages(FailureState& state, HandlerKind handler) +{ + if(handler == HandlerKind::Empty && !state.messagePrimary && state.cancelAt != CancelAt::BeforePlane) + { + return {}; + } + return IFilter::MessageHandler{[&state, handler](const IFilter::Message& message) { + if(message.type == IFilter::Message::Type::Error) + { + ++state.errorCalls; + if(handler == HandlerKind::ErrorThrow) + { + throw PlaneFailure(303); + } + } + if(message.type == IFilter::Message::Type::Progress && message.message.rfind("Identifying Sample Slices: ", 0) == 0) + { + ++state.progressCalls; + if(message.message == fmt::format("Identifying Sample Slices: {}/{}", state.primaryPlane, k_X)) + { + if(state.cancelAt == CancelAt::BeforePlane) + { + state.cancel = true; + state.cancelFired = true; + } + if(state.messagePrimary && state.primary != FaultKind::None) + { + state.primaryFired = true; + (void)RaiseFault(state.primary, true); + } + } + } + }}; +} + +/** + * @brief Fails only the selected mask's guarded diagnostic assembly. + * @param context Points to its FailureState. + * @param mask Identifies the algorithm input at the real assembly boundary. + * @throws AllocationFailure When this exact guard is armed. + */ +void FailDiagnostic(void* context, const IDataArray* mask) +{ + auto& state = *static_cast(context); + if(mask == state.selected) + { + ++state.aggregateCalls; + if(state.aggregateFailure) + { + throw AllocationFailure(404); + } + } +} + +/** + * @struct Outcome + * @brief Captures a lone exception without losing its concrete payload. + */ +struct Outcome +{ + std::optional> result; + FaultKind thrown = FaultKind::None; + int payload = 0; +}; + +/** + * @brief Captures only the test's concrete failure classes; unrelated exceptions remain visible. + * @tparam Callable Callable returning the algorithm Result. + * @param callable Performs one target execution. + * @return Original Result or the selected exception type and payload. + */ +template +Outcome CaptureOutcome(Callable&& callable) +{ + Outcome outcome; + try + { + outcome.result.emplace(callable()); + } catch(const AllocationFailure& error) + { + outcome.thrown = FaultKind::Allocation; + outcome.payload = error.payload; + } catch(const PlaneFailure& error) + { + outcome.thrown = FaultKind::Standard; + outcome.payload = error.payload; + } catch(const NonstandardFailure& error) + { + outcome.thrown = FaultKind::Nonstandard; + outcome.payload = error.payload; + } + return outcome; +} + +/** + * @brief Requires every observed owner, transfer, and reservation scope to be closed. + * @param counts Final observations for the selected mask after the target call ends. + */ +void RequireReleased(const LogicalCounters& counts) +{ + REQUIRE_FALSE(counts.invalid); + REQUIRE_FALSE(counts.invalidExtraAccounting); + REQUIRE(counts.active == 0); + REQUIRE(counts.planeLive == 0); + REQUIRE(counts.xyLive == 0); + REQUIRE(counts.extraLiveBytes == 0); + REQUIRE(counts.extraRetainedBytes == 0); + REQUIRE(counts.allocationEvents == counts.releaseEvents); + REQUIRE(counts.extraAllocationCount == counts.extraReleaseCount); + REQUIRE(counts.extraRetainCount == counts.extraReservationReleaseCount); +} + +/** + * @brief Executes one failure fixture with a full first batch and returns its original carrier. + * @tparam T Bool or UInt8 mask type. + * @param state Supplies faults and records actual operation witnesses. + * @param handler Selects the independent callback condition. + * @param completedPlanes Expected completed X prefix. + * @param successfulZ Expected successful publication Z prefix. + * @return Result or concrete lone exception payload. + */ +template +Outcome RunFailure(FailureState& state, HandlerKind handler, usize completedPlanes, usize successfulZ) +{ + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + auto store = std::make_shared>(state); + DataStructure data = CreateFailureFixture(store); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + state.selected = state.diagnosticMatch ? &mask : nullptr; + auto inputs = Inputs(); + auto messages = Messages(state, handler); + IdentifySampleCCL algorithm(data, messages, state.cancel, &inputs); + LogicalCounters counts; + counts.selected = &mask; + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const BudgetRestore restoreBudget; + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + const auto occupied = budget.reserveWorkingMemory(budget.maximumWorkingMemoryBytes() - 7 * k_Y * k_Z * sizeof(T)); + REQUIRE(occupied.sizeBytes() == budget.maximumWorkingMemoryBytes() - 7 * k_Y * k_Z * sizeof(T)); + auto& collection = DataStoreUtilities::GetIOCollection(); + REQUIRE(collection.begin() != collection.end()); + const auto originalManager = collection.begin()->second; + Outcome outcome; + { + const auto oldObserver = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(oldObserver); }); + REQUIRE(oldObserver.callback == nullptr); + const auto oldDiagnostic = SetIdentifySampleDiagnosticControlForTesting({&state, &FailDiagnostic}); + const auto restoreDiagnostic = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleDiagnosticControlForTesting(oldDiagnostic); }); + REQUIRE(oldDiagnostic.beforeAggregate == nullptr); + const ScopedRecordManager overrideManager(state); + REQUIRE(overrideManager.selected()); + state.armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + if(state.cancelAt == CancelAt::Pre) + { + state.cancel = true; + state.cancelFired = true; + } + outcome = CaptureOutcome([&] { return algorithm(); }); + } + REQUIRE(collection.begin()->second == originalManager); + REQUIRE(mask.getDataStore() == store.get()); + REQUIRE(budget.reservedWorkingMemoryBytes() == occupied.sizeBytes()); + RequireReleased(counts); + REQUIRE(state.liveRecords == 0); + REQUIRE(state.errorCalls == 0); + REQUIRE(state.successfulWrites == successfulZ); + if(state.cancelAt == CancelAt::Pre) + { + REQUIRE(counts.allocationEvents == 0); + REQUIRE(counts.extraRequestCount == 0); + REQUIRE(counts.batchCount == 0); + REQUIRE(state.reads == 0); + REQUIRE(state.writes == 0); + } + else + { + REQUIRE(counts.batchCount == 1); + REQUIRE(counts.maximumBatchWidth == 8); + REQUIRE(counts.extraPeakBytes == 7 * k_Y * k_Z * sizeof(T)); + } + RequirePrefix(data, completedPlanes, successfulZ); + return outcome; +} + +/** + * @brief Maps the scoped fault carrier to the established filter error code. + * @param kind Selects a returned, standard, allocation, or nonstandard failure. + * @param primary True for the plane failure, false for the publication failure. + * @return Original Result code, -272 for allocation failure, or -2 for another exception. + * @pre kind is not FaultKind::None. + */ +int32 ExpectedCode(FaultKind kind, bool primary) +{ + return kind == FaultKind::Result ? (primary ? k_PrimaryCode : k_SecondaryCode) : (kind == FaultKind::Allocation ? -272 : -2); +} + +/** + * @brief Checks original returned diagnostics and original standard-exception text. + * @param error Diagnostic returned by the target call. + * @param kind Identifies the configured failure carrier. + * @param primary True for the plane failure, false for the publication failure. + */ +void RequireOriginalError(const Error& error, FaultKind kind, bool primary) +{ + REQUIRE(error.code == ExpectedCode(kind, primary)); + if(kind == FaultKind::Result) + { + REQUIRE(error.message == (primary ? "primary record Result" : "secondary copy Result")); + } + else + { + REQUIRE(error.message.find("Mask") != std::string::npos); + if(kind == FaultKind::Standard) + { + REQUIRE(error.message.find(primary ? "primary record payload 101" : "secondary copy payload 202") != std::string::npos); + } + } +} + +/** + * @brief Checks both ordered carriers, unchanged warnings, exact locations, and operation witnesses. + * @param outcome Holds the returned combined Result. + * @param state Records the configured faults and the actual operations that reached them. + */ +void RequireDual(const Outcome& outcome, const FailureState& state) +{ + REQUIRE(outcome.thrown == FaultKind::None); + REQUIRE(outcome.result.has_value()); + const auto& result = outcome.result.value(); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 2); + RequireOriginalError(result.errors()[0], state.primary, true); + RequireOriginalError(result.errors()[1], state.secondary, false); + const usize warningCount = static_cast(state.primary == FaultKind::Result) + static_cast(state.secondary == FaultKind::Result); + REQUIRE(result.warnings().size() == warningCount); + usize warning = 0; + if(state.primary == FaultKind::Result) + { + REQUIRE(result.warnings()[warning].code == k_PrimaryWarning); + REQUIRE(result.warnings()[warning++].message == "primary warning"); + } + else + { + const auto& message = result.errors()[0].message; + REQUIRE(message.find(state.messagePrimary ? "reporting plane" : "classifying plane") != std::string::npos); + REQUIRE(message.find("plane 1") != std::string::npos); + } + if(state.secondary == FaultKind::Result) + { + REQUIRE(result.warnings()[warning].code == k_SecondaryWarning); + REQUIRE(result.warnings()[warning].message == "secondary warning"); + } + else + { + const auto& message = result.errors()[1].message; + REQUIRE(message.find(state.secondaryPhase == CopyPhase::CommitRead ? "reading completed prefix" : "writing completed prefix") != std::string::npos); + REQUIRE(message.find(fmt::format("starting at plane 0 with 1 planes at Z {}", state.secondaryZ)) != std::string::npos); + } + REQUIRE(state.primaryFired); + REQUIRE(state.secondaryFired); + REQUIRE(state.aggregateCalls == static_cast(state.diagnosticMatch)); + REQUIRE(state.recordFactories == (state.messagePrimary ? 1 : 2)); + REQUIRE(state.recordReads >= 1); + REQUIRE(state.reads == k_Z + state.secondaryZ + 1); + REQUIRE(state.writes == state.secondaryZ + static_cast(state.secondaryPhase == CopyPhase::CommitWrite)); +} +} // namespace + +TEST_CASE("IdentifySample batch retains both terminal failures in primary order", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto primary = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const auto secondary = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const auto phase = GENERATE(CopyPhase::CommitRead, CopyPhase::CommitWrite); + const auto handler = GENERATE(HandlerKind::Empty, HandlerKind::Capturing, HandlerKind::ErrorThrow); + const bool useBool = GENERATE(false, true); + DYNAMIC_SECTION("primary=" << static_cast(primary) << " secondary=" << static_cast(secondary) << " phase=" << static_cast(phase) << " handler=" << static_cast(handler) + << " Bool=" << useBool) + { + FailureState state; + state.primary = primary; + state.secondary = secondary; + state.secondaryPhase = phase; + const auto outcome = useBool ? RunFailure(state, handler, 1, 1) : RunFailure(state, handler, 1, 1); + RequireDual(outcome, state); + } +} + +TEST_CASE("IdentifySample batch captures a progress callback failure before publication", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto primary = GENERATE(FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const auto secondary = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation); + FailureState state; + state.messagePrimary = true; + state.primary = primary; + state.secondary = secondary; + const auto outcome = RunFailure(state, HandlerKind::ErrorThrow, 1, 1); + RequireDual(outcome, state); + REQUIRE(state.progressCalls == 2); +} + +TEST_CASE("IdentifySample batch preserves lone plane failures and skips an empty prefix", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto kind = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const usize plane = GENERATE(usize{0}, usize{1}); + const bool useBool = GENERATE(false, true); + FailureState state; + state.primary = kind; + state.primaryPlane = plane; + const auto outcome = useBool ? RunFailure(state, HandlerKind::Empty, plane, plane == 0 ? 0 : k_Z) : RunFailure(state, HandlerKind::Empty, plane, plane == 0 ? 0 : k_Z); + REQUIRE(state.primaryFired); + REQUIRE_FALSE(state.secondaryFired); + REQUIRE(state.recordFactories == plane + 1); + REQUIRE(state.aggregateCalls == 0); + REQUIRE(state.reads == (plane == 0 ? k_Z : 2 * k_Z)); + REQUIRE(state.writes == (plane == 0 ? 0 : k_Z)); + if(kind == FaultKind::Result) + { + REQUIRE(outcome.result.has_value()); + const auto& result = outcome.result.value(); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + RequireOriginalError(result.errors()[0], kind, true); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings()[0].code == k_PrimaryWarning); + REQUIRE(result.warnings()[0].message == "primary warning"); + } + else + { + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == kind); + REQUIRE(outcome.payload == 101); + } +} + +TEST_CASE("IdentifySample batch preserves lone gather and commit exceptions", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto phase = GENERATE(CopyPhase::GatherRead, CopyPhase::CommitRead, CopyPhase::CommitWrite); + const auto kind = GENERATE(FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + FailureState state; + state.secondary = kind; + state.secondaryPhase = phase; + state.secondaryZ = 0; + const auto outcome = RunFailure(state, HandlerKind::Empty, phase == CopyPhase::GatherRead ? 0 : 8, 0); + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == kind); + REQUIRE(outcome.payload == 202); + REQUIRE(state.secondaryFired); + REQUIRE_FALSE(state.primaryFired); + REQUIRE(state.aggregateCalls == 0); + REQUIRE(state.recordFactories == (phase == CopyPhase::GatherRead ? 0 : 8)); + REQUIRE(state.reads == (phase == CopyPhase::GatherRead ? 1 : k_Z + 1)); + REQUIRE(state.writes == static_cast(phase == CopyPhase::CommitWrite)); +} + +TEST_CASE("IdentifySample batch preserves primary when diagnostic assembly fails", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto primary = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const auto secondary = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation); + const auto handler = GENERATE(HandlerKind::Empty, HandlerKind::ErrorThrow); + FailureState state; + state.primary = primary; + state.secondary = secondary; + state.aggregateFailure = true; + const auto outcome = RunFailure(state, handler, 1, 1); + REQUIRE(state.primaryFired); + REQUIRE(state.secondaryFired); + REQUIRE(state.aggregateCalls == 1); + REQUIRE(state.recordFactories == 2); + REQUIRE(state.reads == k_Z + 2); + REQUIRE(state.writes == 2); + if(primary == FaultKind::Result) + { + REQUIRE(outcome.result.has_value()); + const auto& result = outcome.result.value(); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + RequireOriginalError(result.errors()[0], primary, true); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings()[0].code == k_PrimaryWarning); + REQUIRE(result.warnings()[0].message == "primary warning"); + } + else + { + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == primary); + REQUIRE(outcome.payload == 101); + } +} + +TEST_CASE("IdentifySample batch cancellation publishes only completed planes", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto point = GENERATE(CancelAt::Pre, CancelAt::Gather, CancelAt::BeforePlane, CancelAt::RecordWrite, CancelAt::CommitRead); + const bool useBool = GENERATE(false, true); + const usize completed = point == CancelAt::CommitRead ? 8 : (point == CancelAt::BeforePlane || point == CancelAt::RecordWrite ? 1 : 0); + FailureState state; + state.cancelAt = point; + const auto outcome = + useBool ? RunFailure(state, HandlerKind::Capturing, completed, completed == 0 ? 0 : k_Z) : RunFailure(state, HandlerKind::Capturing, completed, completed == 0 ? 0 : k_Z); + REQUIRE(state.cancelFired); + REQUIRE(outcome.result.has_value()); + SIMPLNX_RESULT_REQUIRE_VALID(outcome.result.value()); + REQUIRE(state.aggregateCalls == 0); + REQUIRE_FALSE(state.primaryFired); + REQUIRE_FALSE(state.secondaryFired); + REQUIRE(state.recordFactories == (point == CancelAt::RecordWrite ? 2 : completed)); + if(point == CancelAt::Gather) + { + // Cancellation set by the Z=1 read must stop before another gather read. + REQUIRE(state.reads == 2); + REQUIRE(state.writes == 0); + } + if(point == CancelAt::RecordWrite) + { + REQUIRE(state.recordWrites == 2); + } +} + +TEST_CASE("IdentifySample batch failures take precedence over cancellation", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto cancelAt = GENERATE(CancelAt::WithPrimary, CancelAt::WithSecondary); + FailureState state; + state.primary = FaultKind::Result; + state.secondary = FaultKind::Standard; + state.cancelAt = cancelAt; + const auto outcome = RunFailure(state, HandlerKind::Empty, 1, 1); + REQUIRE(state.cancelFired); + RequireDual(outcome, state); +} + +TEST_CASE("IdentifySample lone progress failure still publishes the completed prefix", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto kind = GENERATE(FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const usize plane = GENERATE(usize{0}, usize{1}); + FailureState state; + state.primary = kind; + state.messagePrimary = true; + state.primaryPlane = plane; + const auto outcome = RunFailure(state, HandlerKind::Capturing, plane, plane == 0 ? 0 : k_Z); + REQUIRE(state.primaryFired); + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == kind); + REQUIRE(outcome.payload == 101); + REQUIRE(state.recordFactories == plane); + REQUIRE(state.progressCalls == plane + 1); + REQUIRE(state.aggregateCalls == 0); +} + +TEST_CASE("IdentifySample cancellation cannot hide a lone publication failure", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto kind = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + FailureState state; + state.secondary = kind; + state.secondaryPhase = CopyPhase::CommitWrite; + state.cancelAt = CancelAt::CommitRead; + const auto outcome = RunFailure(state, HandlerKind::Empty, 8, 1); + REQUIRE(state.cancelFired); + REQUIRE(state.secondaryFired); + REQUIRE_FALSE(state.primaryFired); + REQUIRE(state.recordFactories == 8); + REQUIRE(state.aggregateCalls == 0); + REQUIRE(state.reads == k_Z + 2); + REQUIRE(state.writes == 2); + if(kind == FaultKind::Result) + { + REQUIRE(outcome.result.has_value()); + const auto& result = outcome.result.value(); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + RequireOriginalError(result.errors()[0], kind, false); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings()[0].code == k_SecondaryWarning); + REQUIRE(result.warnings()[0].message == "secondary warning"); + } + else + { + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == kind); + REQUIRE(outcome.payload == 202); + } +} + +namespace +{ +/** + * @struct AllocationControl + * @brief Selects the real extra-allocation call to fail, after its reservation exists. + * @note These fixtures use one-byte mask values, so requested values also give the required byte count. + */ +struct AllocationControl +{ + const IDataArray* selected = nullptr; + usize calls = 0; + usize failOn = 1; + usize lastValues = 0; + uint64 occupied = 0; + bool reservationMissing = false; + + /** + * @brief Checks the real reservation before failing the selected extra allocation. + * @param context Points to the live AllocationControl for this execution. + * @param mask Identifies the array passed by the algorithm. + * @param values Number of extra mask values about to be allocated. + * @throws AllocationFailure When the matching array reaches failOn. + */ + static void Before(void* context, const IDataArray* mask, usize values) + { + auto& control = *static_cast(context); + if(mask != control.selected) + { + return; + } + ++control.calls; + control.lastValues = values; + control.reservationMissing = control.reservationMissing || CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes() != control.occupied + values; + if(control.calls == control.failOn) + { + throw AllocationFailure(505); + } + } +}; + +/** + * @struct OwnerObservations + * @brief Checks actual manager state at both extra-owner and token release observations. + */ +struct OwnerObservations +{ + LogicalCounters counts; + uint64 occupied = 0; + bool invalidOrder = false; + + /** + * @brief Checks release order against live manager bytes, then records the logical event. + * @param context Points to the live OwnerObservations for this execution. + * @param mask Identifies the array that produced the event. + * @param event Identifies a transfer, owner, or reservation boundary. + * @param values Number of elements, bytes, or planes as defined by event. + * @param elementBytes Converts values to bytes; admission and width events use one. + * @param success Reports transfer completion; lifetime events require true. + */ + static void Observe(void* context, const IDataArray* mask, IdentifySampleSliceEventForTesting event, usize values, usize elementBytes, bool success) noexcept + { + auto& observation = *static_cast(context); + if(mask != observation.counts.selected) + { + return; + } + const uint64 reserved = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + if(event == IdentifySampleSliceEventForTesting::ExtraPlanesReleased) + { + observation.invalidOrder = observation.invalidOrder || reserved != observation.occupied + values * elementBytes; + } + if(event == IdentifySampleSliceEventForTesting::ExtraReservationReleased) + { + observation.invalidOrder = observation.invalidOrder || reserved != observation.occupied || observation.counts.extraLiveBytes != 0; + } + LogicalCounters::Observe(&observation.counts, mask, event, values, elementBytes, success); + } +}; + +/** + * @brief Verifies the reservation and actual-owner order on early and later allocation failures. + * @tparam T Bool or UInt8 mask type. + * @param failOn One-based real extra-allocation call to fail. + * @param matchIdentity True to arm the selected mask; false to prove identity filtering. + * @param hasHeadroom True to admit eight planes; false to prove the hook is skipped. + */ +template +void CheckAllocationFailure(usize failOn, bool matchIdentity, bool hasHeadroom = true) +{ + static_assert(sizeof(T) == 1); + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + FailureState state; + auto store = std::make_shared>(state); + DataStructure data = CreateFailureFixture(store); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + auto inputs = Inputs(); + const IFilter::MessageHandler messages; + IdentifySampleCCL algorithm(data, messages, state.cancel, &inputs); + AllocationControl control; + control.selected = matchIdentity ? &mask : nullptr; + control.failOn = failOn; + OwnerObservations observations; + observations.counts.selected = &mask; + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const BudgetRestore restoreBudget; + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + const uint64 extraBytes = hasHeadroom ? 7 * k_Y * k_Z * sizeof(T) : 0; + auto occupied = budget.reserveWorkingMemory(budget.maximumWorkingMemoryBytes() - extraBytes); + REQUIRE(occupied.sizeBytes() == budget.maximumWorkingMemoryBytes() - extraBytes); + observations.occupied = occupied.sizeBytes(); + control.occupied = occupied.sizeBytes(); + Outcome outcome; + { + const auto old = SetIdentifySampleExtraAllocationControlForTesting({&control, &AllocationControl::Before}); + const auto restore = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleExtraAllocationControlForTesting(old); }); + REQUIRE(old.beforeAllocate == nullptr); + const auto oldObserver = SetIdentifySampleSliceObserverForTesting({&observations, &OwnerObservations::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(oldObserver); }); + REQUIRE(oldObserver.callback == nullptr); + state.armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + outcome = CaptureOutcome([&] { return algorithm(); }); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == occupied.sizeBytes()); + REQUIRE_FALSE(control.reservationMissing); + REQUIRE_FALSE(observations.invalidOrder); + RequireReleased(observations.counts); + if(matchIdentity && hasHeadroom) + { + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == FaultKind::Allocation); + REQUIRE(outcome.payload == 505); + REQUIRE(control.calls == failOn); + REQUIRE(control.lastValues == 7 * k_Y * k_Z); + REQUIRE(observations.counts.extraRetainCount == failOn); + REQUIRE(observations.counts.extraAllocationCount == failOn - 1); + REQUIRE(observations.counts.batchCount == failOn - 1); + REQUIRE(state.reads == (failOn - 1) * 2 * k_Z); + REQUIRE(state.writes == (failOn - 1) * k_Z); + RequirePrefix(data, (failOn - 1) * 8, failOn == 1 ? 0 : k_Z); + } + else + { + REQUIRE(outcome.result.has_value()); + SIMPLNX_RESULT_REQUIRE_VALID(outcome.result.value()); + REQUIRE(control.calls == 0); + REQUIRE(observations.counts.extraAllocationCount == (hasHeadroom ? 3 : 0)); + REQUIRE(observations.counts.batchCount == (hasHeadroom ? 3 : k_X)); + RequirePrefix(data, k_X, k_Z); + } +} +} // namespace + +TEST_CASE("IdentifySample extra allocation failure releases owners before reservations", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const usize failOn = GENERATE(usize{1}, usize{2}); + const bool useBool = GENERATE(false, true); + const bool matchIdentity = GENERATE(false, true); + if(useBool) + { + CheckAllocationFailure(failOn, matchIdentity); + } + else + { + CheckAllocationFailure(failOn, matchIdentity); + } +} + +TEST_CASE("IdentifySample allocation and diagnostic controls restore nested and thread state", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + AllocationControl outer; + AllocationControl inner; + FailureState outerDiagnostic; + FailureState innerDiagnostic; + const auto originalAllocation = SetIdentifySampleExtraAllocationControlForTesting({&outer, &AllocationControl::Before}); + const auto restoreAllocation = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleExtraAllocationControlForTesting(originalAllocation); }); + const auto originalDiagnostic = SetIdentifySampleDiagnosticControlForTesting({&outerDiagnostic, &FailDiagnostic}); + const auto restoreDiagnostic = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleDiagnosticControlForTesting(originalDiagnostic); }); + REQUIRE(originalAllocation.context == nullptr); + REQUIRE(originalAllocation.beforeAllocate == nullptr); + REQUIRE(originalDiagnostic.context == nullptr); + REQUIRE(originalDiagnostic.beforeAggregate == nullptr); + { + const auto priorAllocation = SetIdentifySampleExtraAllocationControlForTesting({&inner, &AllocationControl::Before}); + const auto restoreInnerAllocation = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleExtraAllocationControlForTesting(priorAllocation); }); + const auto priorDiagnostic = SetIdentifySampleDiagnosticControlForTesting({&innerDiagnostic, &FailDiagnostic}); + const auto restoreInnerDiagnostic = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleDiagnosticControlForTesting(priorDiagnostic); }); + REQUIRE(priorAllocation.context == &outer); + REQUIRE(priorAllocation.beforeAllocate == &AllocationControl::Before); + REQUIRE(priorDiagnostic.context == &outerDiagnostic); + REQUIRE(priorDiagnostic.beforeAggregate == &FailDiagnostic); + } + std::atomic_bool childWasNull = false; + std::thread child([&] { + const auto allocation = SetIdentifySampleExtraAllocationControlForTesting({}); + const auto diagnostic = SetIdentifySampleDiagnosticControlForTesting({}); + childWasNull = allocation.context == nullptr && allocation.beforeAllocate == nullptr && diagnostic.context == nullptr && diagnostic.beforeAggregate == nullptr; + (void)SetIdentifySampleExtraAllocationControlForTesting(allocation); + (void)SetIdentifySampleDiagnosticControlForTesting(diagnostic); + }); + child.join(); + REQUIRE(childWasNull.load()); + const auto currentAllocation = SetIdentifySampleExtraAllocationControlForTesting({}); + const auto currentDiagnostic = SetIdentifySampleDiagnosticControlForTesting({}); + REQUIRE(currentAllocation.context == &outer); + REQUIRE(currentAllocation.beforeAllocate == &AllocationControl::Before); + REQUIRE(currentDiagnostic.context == &outerDiagnostic); + REQUIRE(currentDiagnostic.beforeAggregate == &FailDiagnostic); +} + +TEST_CASE("IdentifySample checked slice dimensions fail before owners and transfers", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + constexpr usize k_SignedMaximum = static_cast(std::numeric_limits::max()); + /** + * @struct InvalidDimensions + * @brief Pairs oversized geometry metadata with its first reachable error code. + */ + struct InvalidDimensions + { + SizeVec3 dimensions; + int32 code; + }; + const usize labelByteBound = std::numeric_limits::max() / sizeof(int64); + std::vector cases{{{std::numeric_limits::max(), 1, 1}, -45464}, {{k_SignedMaximum, 3, 1}, -45463}, {{1, k_SignedMaximum / 2 + 1, 1}, -45464}, + {{k_SignedMaximum / 2 + 1, 2, 1}, -45465}, {{8, 1, k_SignedMaximum / 7 + 1}, -45465}, {{1, labelByteBound / 2 + 1, 1}, -45463}}; + const usize vectorBound = std::vector().max_size(); + REQUIRE(vectorBound <= labelByteBound); + const usize overCapacityRowWidth = vectorBound / 2 + 1; + const bool vectorCapacityReachable = overCapacityRowWidth * 2 <= labelByteBound; + if(vectorCapacityReachable) + { + // Some standard libraries impose a lower vector bound than byte arithmetic does. + cases.push_back({{1, overCapacityRowWidth, 1}, -45465}); + } + else + { + INFO("Vector capacity rejection is preceded by checked label-byte overflow on this standard library"); + } + CAPTURE(vectorBound, labelByteBound, vectorCapacityReachable); + const auto value = GENERATE_COPY(from_range(cases)); + FailureState state; + auto store = std::make_shared>(state); + auto data = CreateFailureFixture(store); + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_ImagePath)); + auto& image = data.getDataRefAs(IdentifySampleBatchTest::k_ImagePath); + const auto dimensions = image.getDimensions(); + const auto restoreDimensions = MakeScopeGuard([&]() noexcept { image.setDimensions(dimensions); }); + image.setDimensions(value.dimensions); + auto inputs = Inputs(); + const IFilter::MessageHandler messages; + IdentifySampleCCL algorithm(data, messages, state.cancel, &inputs); + LogicalCounters counts; + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath)); + counts.selected = &data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath); + const uint64 reserved = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + state.armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + const auto result = algorithm(); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.errors()[0].code == value.code); + REQUIRE(result.errors()[0].message.find("Mask") != std::string::npos); + REQUIRE((result.errors()[0].message.find("bound") != std::string::npos || result.errors()[0].message.find("supported") != std::string::npos || + result.errors()[0].message.find("must fit") != std::string::npos)); + } + REQUIRE(CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes() == reserved); + REQUIRE(state.reads == 0); + REQUIRE(state.writes == 0); + REQUIRE(counts.allocationEvents == 0); + REQUIRE(counts.extraRequestCount == 0); + RequireReleased(counts); + image.setDimensions(dimensions); + RequirePrefix(data, 0, 0); +} + +TEST_CASE("IdentifySample safe zero dimensions retain valid empty work", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const usize zeroAxis = GENERATE(usize{0}, usize{1}, usize{2}); + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + FailureState state; + auto store = std::make_shared>(state); + auto data = CreateFailureFixture(store); + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_ImagePath)); + auto& image = data.getDataRefAs(IdentifySampleBatchTest::k_ImagePath); + const auto original = image.getDimensions(); + auto dimensions = original; + dimensions[zeroAxis] = 0; + const auto restoreDimensions = MakeScopeGuard([&]() noexcept { image.setDimensions(original); }); + image.setDimensions(dimensions); + auto inputs = Inputs(); + const IFilter::MessageHandler messages; + IdentifySampleCCL algorithm(data, messages, state.cancel, &inputs); + LogicalCounters counts; + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath)); + counts.selected = &data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + state.armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + const auto result = algorithm(); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + RequireReleased(counts); + REQUIRE(counts.extraAllocationCount == 0); + if(zeroAxis == 0) + { + REQUIRE(counts.allocationEvents == 0); + REQUIRE(counts.batchCount == 0); + REQUIRE(state.reads == 0); + REQUIRE(state.writes == 0); + } + else + { + REQUIRE(counts.batchCount == k_X); + REQUIRE(counts.minimumBatchWidth == 1); + REQUIRE(counts.maximumBatchWidth == 1); + } + image.setDimensions(original); + RequirePrefix(data, 0, 0); +} + +TEST_CASE("IdentifySample diagnostic control ignores another mask identity", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + FailureState state; + state.primary = FaultKind::Result; + state.secondary = FaultKind::Standard; + state.aggregateFailure = true; + state.diagnosticMatch = false; + const auto outcome = RunFailure(state, HandlerKind::Empty, 1, 1); + RequireDual(outcome, state); + REQUIRE(state.aggregateCalls == 0); +} + +namespace +{ +/** + * @brief Checks actual filter and pipeline Result storage with a dispatch witness. + * @tparam T Bool or UInt8 mask type. + * @param scenario Standard resident algorithm/store scenario. + * @param pipeline True to inspect a PipelineFilter's stored error collection. + * @param primary Selects a returned, standard, or allocation primary failure. + * @param dual True to also fail publication after one successful Z write. + */ +template +void CheckCaller(UnitTest::AlgorithmTestScenario scenario, bool pipeline, FaultKind primary, bool dual) +{ + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + UnitTest::AlgorithmTestScope scope(scenario); + const bool expectCcl = scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore; + FailureState state; + state.primary = expectCcl ? primary : FaultKind::None; + state.secondary = expectCcl && dual ? FaultKind::Standard : FaultKind::None; + if(expectCcl && dual) + { + state.cancelAt = CancelAt::WithPrimary; + } + auto store = std::make_shared>(state); + DataStructure data = CreateFailureFixture(store); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + scope.requireExpectedStore(mask); + IdentifySampleFilter filter; + const auto args = IdentifySampleBatchTest::ArgumentsFor(2, false); + Pipeline owningPipeline; + auto node = std::make_shared(std::make_unique(), args); + REQUIRE(owningPipeline.push_back(node)); + REQUIRE(node->getParentPipeline() == &owningPipeline); + LogicalCounters counts; + counts.selected = &mask; + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const BudgetRestore restoreBudget; + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + auto occupied = budget.reserveWorkingMemory(budget.maximumWorkingMemoryBytes() - 7 * k_Y * k_Z * sizeof(T)); + REQUIRE(occupied.sizeBytes() == budget.maximumWorkingMemoryBytes() - 7 * k_Y * k_Z * sizeof(T)); + auto& collection = DataStoreUtilities::GetIOCollection(); + REQUIRE(collection.begin() != collection.end()); + const auto originalManager = collection.begin()->second; + ErrorCollection errors; + WarningCollection warnings; + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + const ScopedRecordManager overrideManager(state); + REQUIRE(overrideManager.selected()); + state.armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + if(pipeline) + { + const bool success = scope.execute([&] { return node->execute(data, state.cancel); }); + REQUIRE(success == !expectCcl); + errors = node->getErrors(); + warnings = node->getWarnings(); + } + else + { + const auto result = scope.executeFilter(filter, data, args, nullptr, IFilter::MessageHandler{}, state.cancel); + REQUIRE(result.result.valid() == !expectCcl); + if(result.result.invalid()) + { + errors = result.result.errors(); + } + warnings = result.result.warnings(); + } + } + REQUIRE(collection.begin()->second == originalManager); + REQUIRE(mask.getDataStore() == store.get()); + REQUIRE(budget.reservedWorkingMemoryBytes() == occupied.sizeBytes()); + RequireReleased(counts); + REQUIRE(state.liveRecords == 0); + if(expectCcl) + { + REQUIRE(state.primaryFired); + REQUIRE(state.secondaryFired == dual); + REQUIRE(state.recordFactories == 2); + REQUIRE(errors.size() == (dual ? 2 : 1)); + REQUIRE(errors[0].code == ExpectedCode(primary, true)); + if(primary == FaultKind::Result) + { + REQUIRE(errors[0].message == "primary record Result"); + REQUIRE(warnings.size() == 1); + REQUIRE(warnings[0].code == k_PrimaryWarning); + REQUIRE(warnings[0].message == "primary warning"); + } + else + { + REQUIRE(warnings.empty()); + if(primary == FaultKind::Standard) + { + REQUIRE(errors[0].message.find("primary record payload 101") != std::string::npos); + } + } + if(dual) + { + REQUIRE(state.cancelFired); + REQUIRE(errors[1].code == -2); + REQUIRE(errors[1].message.find("secondary copy payload 202") != std::string::npos); + REQUIRE(errors[1].message.find("writing completed prefix starting at plane 0 with 1 planes at Z 1") != std::string::npos); + } + REQUIRE(state.writes == (dual ? 2 : k_Z)); + RequirePrefix(data, 1, dual ? 1 : k_Z); + } + else + { + REQUIRE(errors.empty()); + REQUIRE(warnings.empty()); + REQUIRE_FALSE(state.primaryFired); + REQUIRE_FALSE(state.secondaryFired); + REQUIRE(state.recordFactories == 0); + RequirePrefix(data, k_X, k_Z); + } +} +} // namespace + +TEST_CASE("IdentifySample filter and pipeline retain failure Results without Output callbacks", "[SimplnxCore][IdentifySampleFilter][.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool pipeline = GENERATE(false, true); + const auto primary = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation); + const bool dual = GENERATE(false, true); + const bool useBool = GENERATE(false, true); + DYNAMIC_SECTION("scenario=" << scenario << " pipeline=" << pipeline << " primary=" << static_cast(primary) << " dual=" << dual << " Bool=" << useBool) + { + if(useBool) + { + CheckCaller(scenario, pipeline, primary, dual); + } + else + { + CheckCaller(scenario, pipeline, primary, dual); + } + } +} + +TEST_CASE("IdentifySample zero extra grant does not call its allocation control", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + CheckAllocationFailure(1, true, false); +} + +TEST_CASE("IdentifySample orthogonal CCL owns only its used plane carrier", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto plane = GENERATE(ChoicesParameter::ValueType{0}, ChoicesParameter::ValueType{1}); + const bool useBool = GENERATE(false, true); + const bool fillHoles = GENERATE(false, true); + const auto check = [&]() { + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + DataStructure data = IdentifySampleBatchTest::CreateFixture(plane); + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath)); + LogicalCounters counts; + counts.selected = &data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath); + auto inputs = Inputs(); + inputs.SliceBySlicePlaneIndex = plane; + inputs.FillHoles = fillHoles; + const IFilter::MessageHandler messages; + const std::atomic_bool cancel = false; + IdentifySampleCCL algorithm(data, messages, cancel, &inputs); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restore = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + const auto result = algorithm(); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + RequireReleased(counts); + REQUIRE(counts.allocationEvents == 1); + REQUIRE(counts.namedCarrierPeak == 11 * 13 * sizeof(T)); + REQUIRE(counts.extraRequestCount == 0); + REQUIRE(counts.batchCount == 0); + REQUIRE(counts.reads == (plane == 0 ? 19 : 19 * 13)); + REQUIRE(counts.writes == counts.reads); + REQUIRE(counts.readSuccess == counts.reads); + REQUIRE(counts.writeSuccess == counts.writes); + IdentifySampleBatchTest::RequireOutput(data, plane, fillHoles); + }; + if(useBool) + { + check.template operator()(); + } + else + { + check.template operator()(); + } +} + +TEST_CASE("IdentifySample filter pre-cancellation stops before algorithm dispatch", "[SimplnxCore][IdentifySampleFilter][.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const bool useBool = GENERATE(false, true); + const auto check = [&]() { + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + FailureState state; + auto store = std::make_shared>(state); + DataStructure data = CreateFailureFixture(store); + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs(IdentifySampleBatchTest::k_MaskPath); + LogicalCounters counts; + counts.selected = &mask; + IdentifySampleFilter filter; + const auto args = IdentifySampleBatchTest::ArgumentsFor(2, false); + const uint64 reserved = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + const auto priorDispatch = GetAlgorithmPathExecutionCounts(); + const auto restoreDispatch = MakeScopeGuard([&]() noexcept { SetAlgorithmPathExecutionCounts(priorDispatch); }); + ResetAlgorithmPathExecutionCounts(); + state.armed = true; + state.cancel = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + // Preflight rejects this call before the selector, so no execution witness is required. + const auto result = filter.execute(data, args, nullptr, IFilter::MessageHandler{}, state.cancel); + SIMPLNX_RESULT_REQUIRE_INVALID(result.result); + REQUIRE(result.result.errors().size() == 1); + REQUIRE(result.result.errors()[0].code == -1); + REQUIRE(result.result.warnings().empty()); + const auto dispatch = GetAlgorithmPathExecutionCounts(); + REQUIRE(dispatch.InCore == 0); + REQUIRE(dispatch.OutOfCore == 0); + REQUIRE(dispatch.InCoreOnInMemoryStore == 0); + REQUIRE(dispatch.InCoreOnOutOfCoreStore == 0); + REQUIRE(dispatch.OutOfCoreOnInMemoryStore == 0); + REQUIRE(dispatch.OutOfCoreOnOutOfCoreStore == 0); + } + REQUIRE(mask.getIDataStore() == store.get()); + REQUIRE(CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes() == reserved); + REQUIRE(state.reads == 0); + REQUIRE(state.writes == 0); + REQUIRE(counts.allocationEvents == 0); + REQUIRE(counts.extraRequestCount == 0); + RequireReleased(counts); + RequirePrefix(data, 0, 0); + }; + if(useBool) + { + check.template operator()(); + } + else + { + check.template operator()(); + } +} + +namespace +{ +/** + * @enum OrthogonalStop + * @brief Selects a real XY or XZ transfer or message boundary on the second plane. + */ +enum class OrthogonalStop +{ + ReadFailure, ///< Refuses the selected read before any values are copied. + WriteFailure, ///< Refuses the selected write before any values are changed. + ReadCancel, ///< Sets cancellation after the selected successful read. + MessageCancel, ///< Sets cancellation after the first plane's completed progress message. + WriteCancel ///< Sets cancellation after the second plane's first successful write. +}; + +/** + * @struct OrthogonalTransfer + * @brief Records one actual resident bulk call, including an incomplete call that throws. + */ +struct OrthogonalTransfer +{ + bool write = false; + usize start = 0; + usize count = 0; + bool success = false; +}; + +/** + * @struct OrthogonalTransferState + * @brief Arms deterministic controls after fixture setup and keeps a bounded call trace. + * @note The state outlives the reporting store and all callbacks. Tests execute serially. + */ +struct OrthogonalTransferState +{ + OrthogonalStop stop = OrthogonalStop::ReadFailure; + FaultKind fault = FaultKind::None; + usize triggerCopy = 0; + bool armed = false; + bool fired = false; + usize reads = 0; + usize writes = 0; + usize transferCount = 0; + usize sliceMessages = 0; + bool sliceOrderMatches = true; + std::atomic_bool cancel = false; + // Two XZ planes require thirteen read rows and thirteen write rows each. + std::array transfers{}; + + /** + * @brief Records the actual requested range before the selected operation can fail. + * @param write True for a mask write, false for a mask read. + * @param start First flat value in the store. + * @param count Number of requested values. + * @return Trace entry whose success flag the real copy result sets. + */ + OrthogonalTransfer& beginTransfer(bool write, usize start, usize count) + { + REQUIRE(transferCount < transfers.size()); + auto& transfer = transfers[transferCount++]; + transfer = {write, start, count, false}; + return transfer; + } +}; + +/** + * @class OrthogonalMaskStore + * @brief Reports real resident copies and injects a fault or cancellation at one selected call. + * @tparam T Bool or UInt8 mask element type. + */ +template +class OrthogonalMaskStore : public DataStore +{ +public: + /** + * @brief Creates a resident store with the requested fixture shape. + * @param tupleShape Matches the image dimensions in Z, Y, X order. + * @param state Supplies controls and receives actual transfers; outlives this store. + */ + OrthogonalMaskStore(const ShapeType& tupleShape, OrthogonalTransferState& state) + : DataStore(tupleShape, ShapeType{1}, std::optional{}) + , m_State(state) + { + } + + ~OrthogonalMaskStore() override = default; + OrthogonalMaskStore(const OrthogonalMaskStore&) = default; + OrthogonalMaskStore(OrthogonalMaskStore&&) = default; + OrthogonalMaskStore& operator=(const OrthogonalMaskStore&) = delete; + OrthogonalMaskStore& operator=(OrthogonalMaskStore&&) = delete; + + /** + * @brief Copies source values unless the selected read refuses the transfer. + * @param start First flat mask value. + * @param values Receives the requested mask values. + * @return Real copy Result or the selected error and warning. + * @throws PlaneFailure For a selected standard failure. + * @throws AllocationFailure For a selected allocation failure. + * @throws NonstandardFailure For a selected nonstandard failure. + */ + Result<> copyIntoBuffer(usize start, nonstd::span values) const override + { + if(!m_State.armed) + { + return DataStore::copyIntoBuffer(start, values); + } + auto& transfer = m_State.beginTransfer(false, start, values.size()); + const usize call = ++m_State.reads; + if(m_State.stop == OrthogonalStop::ReadFailure && call == m_State.triggerCopy) + { + m_State.fired = true; + return RaiseFault(m_State.fault, false); + } + auto result = DataStore::copyIntoBuffer(start, values); + transfer.success = result.valid(); + if(result.valid() && m_State.stop == OrthogonalStop::ReadCancel && call == m_State.triggerCopy) + { + m_State.fired = true; + m_State.cancel = true; + } + return result; + } + + /** + * @brief Copies output values unless the selected write refuses the transfer. + * @param start First flat mask value. + * @param values Supplies replacement mask values. + * @return Real copy Result or the selected error and warning. + * @throws PlaneFailure For a selected standard failure. + * @throws AllocationFailure For a selected allocation failure. + * @throws NonstandardFailure For a selected nonstandard failure. + */ + Result<> copyFromBuffer(usize start, nonstd::span values) override + { + if(!m_State.armed) + { + return DataStore::copyFromBuffer(start, values); + } + auto& transfer = m_State.beginTransfer(true, start, values.size()); + const usize call = ++m_State.writes; + if(m_State.stop == OrthogonalStop::WriteFailure && call == m_State.triggerCopy) + { + m_State.fired = true; + return RaiseFault(m_State.fault, false); + } + auto result = DataStore::copyFromBuffer(start, values); + transfer.success = result.valid(); + if(result.valid() && m_State.stop == OrthogonalStop::WriteCancel && call == m_State.triggerCopy) + { + m_State.fired = true; + m_State.cancel = true; + } + return result; + } + +private: + OrthogonalTransferState& m_State; +}; + +/** + * @brief Checks all mask values against independent initial and processed plane predicates. + * @tparam T Bool or UInt8 mask element type. + * @param data Contains the mask after instrumentation is disarmed. + * @param plane XY=0 or XZ=1. + * @param completedPlanes Number of complete output planes before the partial plane. + * @param partialRows Number of successfully written rows in the next XZ plane. + */ +template +void RequireOrthogonalPrefix(const DataStructure& data, ChoicesParameter::ValueType plane, usize completedPlanes, usize partialRows) +{ + const auto dimensions = IdentifySampleBatchTest::Dimensions(plane); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + const auto& store = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath).getDataStoreRef(); + const usize xyValues = dimensions[0] * dimensions[1]; + auto xy = std::make_unique(xyValues); + for(usize z = 0; z < dimensions[2]; ++z) + { + const auto result = store.copyIntoBuffer(z * xyValues, nonstd::span(xy.get(), xyValues)); + SIMPLNX_RESULT_REQUIRE_VALID(result); + for(usize y = 0; y < dimensions[1]; ++y) + { + for(usize x = 0; x < dimensions[0]; ++x) + { + const auto coordinates = IdentifySampleBatchTest::PlaneCoordinates(plane, x, y, z); + const usize u = coordinates[0]; + const usize v = coordinates[1]; + const usize fixed = coordinates[2]; + const bool processed = fixed < completedPlanes || (plane == 1 && fixed == completedPlanes && v < partialRows); + const bool initial = IdentifySampleBatchTest::InputValue(u, v, fixed) || (u == 0 && v == 0); + const bool expected = processed ? IdentifySampleBatchTest::ExpectedValue(u, v, fixed, false) : initial; + INFO("plane=" << plane << " xyz=" << x << ',' << y << ',' << z << " complete=" << completedPlanes << " partial rows=" << partialRows); + REQUIRE(xy[y * dimensions[0] + x] == static_cast(expected)); + } + } + } + UnitTest::CheckArraysInheritTupleDims(data); +} + +/** + * @brief Runs one direct CCL case and checks its complete transfer trace and output. + * @tparam T Bool or UInt8 mask element type. + * @param plane XY=0 or XZ=1. + * @param stop Chooses the second plane's failure or cancellation boundary. + * @param fault Result or exception carrier for a transfer failure; None for cancellation. + * @return Original Result or concrete exception payload. + */ +template +Outcome RunOrthogonalTransfer(ChoicesParameter::ValueType plane, OrthogonalStop stop, FaultKind fault) +{ + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + const auto dimensions = IdentifySampleBatchTest::Dimensions(plane); + const usize copiesPerPlane = plane == 0 ? 1 : dimensions[2]; + OrthogonalTransferState state; + state.stop = stop; + state.fault = fault; + state.triggerCopy = copiesPerPlane + (plane == 1 && stop != OrthogonalStop::WriteCancel ? 2 : 1); + auto store = std::make_shared>(ShapeType{dimensions[2], dimensions[1], dimensions[0]}, state); + DataStructure data = IdentifySampleBatchTest::CreateFixture(plane, 19, store); + // The early isolated cell makes a successful first-row write distinguishable from unchanged input. + for(usize fixed = 0; fixed < 19; ++fixed) + { + const usize offset = plane == 0 ? fixed * dimensions[0] * dimensions[1] : fixed * dimensions[0]; + store->setValue(offset, static_cast(true)); + } + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + const auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + REQUIRE(mask.getIDataStore() == store.get()); + REQUIRE(store->getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE_FALSE(store->getChunkShape().has_value()); + auto inputs = Inputs(); + inputs.SliceBySlicePlaneIndex = plane; + const IFilter::MessageHandler messages{[&state](const IFilter::Message& message) { + if(message.type == IFilter::Message::Type::Progress && message.message.rfind("Identifying Sample Slices: ", 0) == 0) + { + state.sliceOrderMatches = state.sliceOrderMatches && message.message.starts_with(fmt::format("Identifying Sample Slices: {}/19", state.sliceMessages + 1)); + ++state.sliceMessages; + if(state.stop == OrthogonalStop::MessageCancel && message.message.starts_with("Identifying Sample Slices: 1/19")) + { + state.fired = true; + state.cancel = true; + } + } + }}; + IdentifySampleCCL algorithm(data, messages, state.cancel, &inputs); + LogicalCounters counts; + counts.selected = &mask; + FailureState recordState; + auto& collection = DataStoreUtilities::GetIOCollection(); + REQUIRE(collection.begin() != collection.end()); + const auto originalManager = collection.begin()->second; + const uint64 reserved = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + Outcome outcome; + std::string exceptionMessage; + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counts, &LogicalCounters::Observe}); + const auto restoreObserver = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + const ScopedRecordManager overrideManager(recordState); + REQUIRE(overrideManager.selected()); + state.armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { state.armed = false; }); + outcome = CaptureOutcome([&] { + try + { + return algorithm(); + } catch(const PlaneFailure& error) + { + exceptionMessage = error.what(); + throw; + } + }); + } + REQUIRE_FALSE(state.armed); + REQUIRE(collection.begin()->second == originalManager); + REQUIRE(CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes() == reserved); + REQUIRE(recordState.liveRecords == 0); + REQUIRE(state.fired); + REQUIRE(state.sliceOrderMatches); + REQUIRE(state.sliceMessages == (stop == OrthogonalStop::WriteCancel ? 2 : 1)); + const bool readFailure = stop == OrthogonalStop::ReadFailure; + const bool writeFailure = stop == OrthogonalStop::WriteFailure; + const bool writeCancel = stop == OrthogonalStop::WriteCancel; + REQUIRE(state.cancel.load() == (!readFailure && !writeFailure)); + REQUIRE(recordState.recordFactories == (writeFailure || writeCancel ? 2 : 1)); + if(fault == FaultKind::Standard) + { + REQUIRE(exceptionMessage == "secondary copy payload 202"); + } + + const usize secondReadCount = stop == OrthogonalStop::MessageCancel ? 0 : (readFailure ? (plane == 0 ? 1 : 2) : copiesPerPlane); + const usize secondWriteCount = writeFailure ? (plane == 0 ? 1 : 2) : (writeCancel ? copiesPerPlane : 0); + REQUIRE(state.reads == copiesPerPlane + secondReadCount); + REQUIRE(state.writes == copiesPerPlane + secondWriteCount); + REQUIRE(state.transferCount == state.reads + state.writes); + usize transferIdx = 0; + for(usize fixed = 0; fixed < 2; ++fixed) + { + for(const bool write : {false, true}) + { + const usize copyCount = fixed == 0 ? copiesPerPlane : (write ? secondWriteCount : secondReadCount); + for(usize row = 0; row < copyCount; ++row) + { + REQUIRE(transferIdx < state.transferCount); + const auto& transfer = state.transfers[transferIdx++]; + const usize expectedStart = plane == 0 ? fixed * dimensions[0] * dimensions[1] : row * dimensions[0] * dimensions[1] + fixed * dimensions[0]; + const usize expectedCount = plane == 0 ? dimensions[0] * dimensions[1] : dimensions[0]; + const bool refused = fixed == 1 && row + 1 == copyCount && (write ? writeFailure : readFailure); + CAPTURE(fixed, write, row); + REQUIRE(transfer.write == write); + REQUIRE(transfer.start == expectedStart); + REQUIRE(transfer.count == expectedCount); + REQUIRE(transfer.success == !refused); + } + } + } + REQUIRE(transferIdx == state.transferCount); + RequireReleased(counts); + REQUIRE(counts.allocationEvents == 1); + REQUIRE(counts.namedCarrierPeak == 11 * 13 * sizeof(T)); + REQUIRE(counts.extraRequestCount == 0); + REQUIRE(counts.batchCount == 0); + REQUIRE(counts.reads == state.reads); + REQUIRE(counts.writes == state.writes); + REQUIRE(counts.readSuccess == state.reads - static_cast(readFailure)); + REQUIRE(counts.writeSuccess == state.writes - static_cast(writeFailure)); + // A failed XZ write retains the first row. Write cancellation lets every row of that plane finish. + RequireOrthogonalPrefix(data, plane, writeCancel ? 2 : 1, writeFailure && plane == 1 ? 1 : 0); + return outcome; +} +} // namespace + +TEST_CASE("IdentifySample orthogonal transfer failures preserve exact completed values", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto plane = GENERATE(ChoicesParameter::ValueType{0}, ChoicesParameter::ValueType{1}); + const auto stop = GENERATE(OrthogonalStop::ReadFailure, OrthogonalStop::WriteFailure); + const auto fault = GENERATE(FaultKind::Result, FaultKind::Standard, FaultKind::Allocation, FaultKind::Nonstandard); + const bool useBool = GENERATE(false, true); + DYNAMIC_SECTION("plane=" << plane << " stop=" << static_cast(stop) << " fault=" << static_cast(fault) << " Bool=" << useBool) + { + const auto outcome = useBool ? RunOrthogonalTransfer(plane, stop, fault) : RunOrthogonalTransfer(plane, stop, fault); + if(fault == FaultKind::Result) + { + REQUIRE(outcome.thrown == FaultKind::None); + REQUIRE(outcome.result.has_value()); + const auto& result = outcome.result.value(); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.errors()[0].code == k_SecondaryCode); + REQUIRE(result.errors()[0].message == "secondary copy Result"); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings()[0].code == k_SecondaryWarning); + REQUIRE(result.warnings()[0].message == "secondary warning"); + } + else + { + REQUIRE_FALSE(outcome.result.has_value()); + REQUIRE(outcome.thrown == fault); + REQUIRE(outcome.payload == 202); + } + } +} + +TEST_CASE("IdentifySample orthogonal cancellation preserves per-plane publication", "[.IdentifySampleObservation][.IdentifySampleBatchFailures]") +{ + UnitTest::LoadPlugins(); + const auto plane = GENERATE(ChoicesParameter::ValueType{0}, ChoicesParameter::ValueType{1}); + const auto stop = GENERATE(OrthogonalStop::ReadCancel, OrthogonalStop::MessageCancel, OrthogonalStop::WriteCancel); + const bool useBool = GENERATE(false, true); + DYNAMIC_SECTION("plane=" << plane << " stop=" << static_cast(stop) << " Bool=" << useBool) + { + const auto outcome = useBool ? RunOrthogonalTransfer(plane, stop, FaultKind::None) : RunOrthogonalTransfer(plane, stop, FaultKind::None); + REQUIRE(outcome.thrown == FaultKind::None); + REQUIRE(outcome.result.has_value()); + const auto& result = outcome.result.value(); + SIMPLNX_RESULT_REQUIRE_VALID(result); + REQUIRE(result.warnings().empty()); + } +} diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleBatchTestUtilities.hpp b/src/Plugins/SimplnxCore/test/IdentifySampleBatchTestUtilities.hpp new file mode 100644 index 0000000000..cf78b6246c --- /dev/null +++ b/src/Plugins/SimplnxCore/test/IdentifySampleBatchTestUtilities.hpp @@ -0,0 +1,300 @@ +#pragma once + +#include "SimplnxCore/Filters/IdentifySampleFilter.hpp" + +#include "simplnx/Core/Application.hpp" +#include "simplnx/Core/Preferences.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/Geometry/ImageGeom.hpp" +#include "simplnx/Parameters/ChoicesParameter.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" + +#include + +#include +#include +#include + +namespace IdentifySampleBatchTest +{ +/** + * @namespace IdentifySampleBatchTest + * @brief Supplies independent mask fixtures for resident and HDF5 Identify Sample tests. + */ +using namespace nx::core; + +/** + * @brief Identifies the geometry created by the fixture. + */ +inline const DataPath k_ImagePath{{"BatchImage"}}; +/** + * @brief Identifies the mask updated by the filter. + */ +inline const DataPath k_MaskPath{{"BatchImage", "CellData", "Mask"}}; + +/** + * @class ScopedPreferenceRestore + * @brief Preserves explicit-key presence and JSON number types around the scenario scopes. + */ +class ScopedPreferenceRestore +{ +public: + /** @brief Captures both preference values before a scenario changes them. */ + ScopedPreferenceRestore() + : m_Preferences(*Application::GetOrCreateInstance()->getPreferences()) + , m_SizePresent(m_Preferences.contains(Preferences::k_LargeDataSize_Key)) + , m_ModePresent(m_Preferences.contains(Preferences::k_DataStorageMode_Key)) + , m_Size(m_Preferences.value(Preferences::k_LargeDataSize_Key)) + , m_Mode(m_Preferences.value(Preferences::k_DataStorageMode_Key)) + { + } + + /** @brief Restores the captured values and explicit-key presence without saving a file. */ + ~ScopedPreferenceRestore() + { + // Recompute the dependent default before restoring an absent threshold key. + m_Preferences.setValue(Preferences::k_LargeDataSize_Key, m_Size); + if(!m_SizePresent) + { + m_Preferences.removeValue(Preferences::k_LargeDataSize_Key); + } + if(m_ModePresent) + { + m_Preferences.setValue(Preferences::k_DataStorageMode_Key, m_Mode); + } + else + { + m_Preferences.removeValue(Preferences::k_DataStorageMode_Key); + } + } + + ScopedPreferenceRestore(const ScopedPreferenceRestore&) = delete; + ScopedPreferenceRestore(ScopedPreferenceRestore&&) = delete; + ScopedPreferenceRestore& operator=(const ScopedPreferenceRestore&) = delete; + ScopedPreferenceRestore& operator=(ScopedPreferenceRestore&&) = delete; + +private: + Preferences& m_Preferences; + bool m_SizePresent; + bool m_ModePresent; + nlohmann::json m_Size; + nlohmann::json m_Mode; +}; + +/** + * @brief Rotates nineteen 11 by 13 planes through the three supported orientations. + * @param plane XY=0, XZ=1, or YZ=2. + * @param fixedCount Number of planes along the remaining axis. + * @return Geometry dimensions in X, Y, Z order. + */ +inline SizeVec3 Dimensions(ChoicesParameter::ValueType plane, usize fixedCount = 19) +{ + REQUIRE(fixedCount > 0); + switch(plane) + { + case 0: + return {11, 13, fixedCount}; + case 1: + return {11, fixedCount, 13}; + case 2: + return {fixedCount, 11, 13}; + default: + FAIL("The fixture supports only XY, XZ, and YZ planes"); + return {}; + } +} + +/** + * @brief Maps geometry coordinates to the independent plane predicate. + * @param plane XY=0, XZ=1, or YZ=2. + * @param x X coordinate. + * @param y Y coordinate. + * @param z Z coordinate. + * @return Coordinates u, v, and fixed-plane index. + */ +inline std::array PlaneCoordinates(ChoicesParameter::ValueType plane, usize x, usize y, usize z) +{ + switch(plane) + { + case 0: + return {x, y, z}; + case 1: + return {x, z, y}; + case 2: + return {y, z, x}; + default: + FAIL("The fixture supports only XY, XZ, and YZ planes"); + return {}; + } +} + +/** + * @brief Defines a 47-cell component, an enclosed false cell, and one isolated true cell. + * @param u First plane coordinate. + * @param v Second plane coordinate. + * @param fixed Plane index that selects the alternating rectangle position. + * @return Initial mask value. + */ +inline bool InputValue(usize u, usize v, usize fixed) +{ + const usize shift = fixed % 2; + const bool rectangle = u >= 2 + shift && u <= 7 + shift && v >= 2 && v <= 9; + const bool hole = u == 4 + shift && v == 5; + const bool isolated = u == 0 && v == 12; + return (rectangle && !hole) || isolated; +} + +/** + * @brief Defines the retained rectangle independently of the filter output. + * @param u First plane coordinate. + * @param v Second plane coordinate. + * @param fixed Plane index that selects the alternating rectangle position. + * @param fillHoles True to include the enclosed false cell. + * @return Expected mask value after the isolated component is removed. + */ +inline bool ExpectedValue(usize u, usize v, usize fixed, bool fillHoles) +{ + const usize shift = fixed % 2; + return u >= 2 + shift && u <= 7 + shift && v >= 2 && v <= 9 && (fillHoles || u != 4 + shift || v != 5); +} + +/** + * @brief Builds the mask through the configured factory unless a resident reporting store is supplied. + * @tparam T Bool or UInt8 mask element type. + * @param plane XY=0, XZ=1, or YZ=2. + * @param fixedCount Number of independent planes. + * @param reportingStore Optional resident store with matching tuple and component dimensions. + * @return Geometry and mask initialized through bounded XY transfers. + */ +template +DataStructure CreateFixture(ChoicesParameter::ValueType plane, usize fixedCount = 19, std::shared_ptr> reportingStore = {}) +{ + const auto dimensions = Dimensions(plane, fixedCount); + const ShapeType tupleShape{dimensions[2], dimensions[1], dimensions[0]}; + DataStructure dataStructure; + auto* image = ImageGeom::Create(dataStructure, k_ImagePath.getTargetName()); + REQUIRE(image != nullptr); + image->setDimensions(dimensions); + image->setSpacing({1.0F, 1.0F, 1.0F}); + image->setOrigin({0.0F, 0.0F, 0.0F}); + auto* cellData = AttributeMatrix::Create(dataStructure, "CellData", tupleShape, image->getId()); + REQUIRE(cellData != nullptr); + image->setCellData(*cellData); + auto store = reportingStore ? std::move(reportingStore) : DataStoreUtilities::CreateDataStore(dataStructure, k_MaskPath, tupleShape, ShapeType{1}); + REQUIRE(store != nullptr); + REQUIRE(store->getTupleShape() == tupleShape); + REQUIRE(store->getComponentShape() == ShapeType{1}); + auto* mask = DataArray::Create(dataStructure, "Mask", store, cellData->getId()); + REQUIRE(mask != nullptr); + + const usize xySize = dimensions[0] * dimensions[1]; + auto xy = std::make_unique(xySize); + for(usize z = 0; z < dimensions[2]; z++) + { + for(usize y = 0; y < dimensions[1]; y++) + { + for(usize x = 0; x < dimensions[0]; x++) + { + const auto coordinates = PlaneCoordinates(plane, x, y, z); + xy[y * dimensions[0] + x] = static_cast(InputValue(coordinates[0], coordinates[1], coordinates[2])); + } + } + const auto writeResult = store->copyFromBuffer(z * xySize, nonstd::span(xy.get(), xySize)); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + } + return dataStructure; +} + +/** + * @brief Selects slice processing for the fixture mask. + * @param plane XY=0, XZ=1, or YZ=2. + * @param fillHoles Selects hole filling. + * @return Complete filter arguments. + */ +inline Arguments ArgumentsFor(ChoicesParameter::ValueType plane, bool fillHoles) +{ + Arguments args; + args.insert(IdentifySampleFilter::k_SelectedImageGeometryPath_Key, std::make_any(k_ImagePath)); + args.insert(IdentifySampleFilter::k_MaskArrayPath_Key, std::make_any(k_MaskPath)); + args.insert(IdentifySampleFilter::k_FillHoles_Key, std::make_any(fillHoles)); + args.insert(IdentifySampleFilter::k_SliceBySlice_Key, std::make_any(true)); + args.insert(IdentifySampleFilter::k_SliceBySlicePlane_Key, std::make_any(plane)); + return args; +} + +/** + * @brief Checks every output cell against a rectangle predicate that does not call production classification code. + * @tparam T Bool or UInt8 mask element type. + * @param dataStructure Filter output. + * @param plane XY=0, XZ=1, or YZ=2. + * @param fillHoles True when the enclosed false cell must become true. + * @param fixedCount Number of independent planes. + */ +template +void RequireOutput(const DataStructure& dataStructure, ChoicesParameter::ValueType plane, bool fillHoles, usize fixedCount = 19) +{ + const auto dimensions = Dimensions(plane, fixedCount); + const ShapeType tupleShape{dimensions[2], dimensions[1], dimensions[0]}; + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_ImagePath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_MaskPath.getParent())); + REQUIRE_NOTHROW(dataStructure.getDataRefAs>(k_MaskPath)); + const auto& image = dataStructure.getDataRefAs(k_ImagePath); + const auto& cellData = dataStructure.getDataRefAs(k_MaskPath.getParent()); + const auto& mask = dataStructure.getDataRefAs>(k_MaskPath); + REQUIRE(image.getDimensions() == dimensions); + REQUIRE(image.getSpacing() == FloatVec3(1.0F, 1.0F, 1.0F)); + REQUIRE(image.getOrigin() == FloatVec3(0.0F, 0.0F, 0.0F)); + REQUIRE(cellData.getShape() == tupleShape); + REQUIRE(mask.getTupleShape() == tupleShape); + REQUIRE(mask.getComponentShape() == ShapeType{1}); + + const usize xySize = dimensions[0] * dimensions[1]; + REQUIRE(mask.getNumberOfTuples() == xySize * dimensions[2]); + auto xy = std::make_unique(xySize); + usize trueCount = 0; + for(usize z = 0; z < dimensions[2]; z++) + { + const auto readResult = mask.getDataStoreRef().copyIntoBuffer(z * xySize, nonstd::span(xy.get(), xySize)); + SIMPLNX_RESULT_REQUIRE_VALID(readResult); + for(usize y = 0; y < dimensions[1]; y++) + { + for(usize x = 0; x < dimensions[0]; x++) + { + const auto coordinates = PlaneCoordinates(plane, x, y, z); + const bool expected = ExpectedValue(coordinates[0], coordinates[1], coordinates[2], fillHoles); + INFO("plane=" << plane << " fill=" << fillHoles << " xyz=" << x << ',' << y << ',' << z); + REQUIRE(xy[y * dimensions[0] + x] == static_cast(expected)); + trueCount += static_cast(xy[y * dimensions[0] + x]); + } + } + } + REQUIRE(trueCount == fixedCount * (fillHoles ? 48 : 47)); + nx::core::UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +/** + * @brief Runs a fixture under the caller's witnessed resident or actual-OOC scenario. + * @tparam T Bool or UInt8 mask element type. + * @tparam ScopeT Public or private algorithm scenario scope. + * @param scope Selects and verifies algorithm and store dispatch. + * @param dataStructure Initialized mask fixture. + * @param plane XY=0, XZ=1, or YZ=2. + * @param fillHoles Selects hole filling. + * @param fixedCount Number of independent planes. + */ +template +void ExecuteAndRequire(ScopeT& scope, DataStructure& dataStructure, ChoicesParameter::ValueType plane, bool fillHoles, usize fixedCount = 19) +{ + REQUIRE_NOTHROW(dataStructure.getDataRefAs>(k_MaskPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs>(k_MaskPath)); + IdentifySampleFilter filter; + const auto args = ArgumentsFor(plane, fillHoles); + const auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + const auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + RequireOutput(dataStructure, plane, fillHoles, fixedCount); +} +} // namespace IdentifySampleBatchTest diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleBenchmarkUtilities.hpp b/src/Plugins/SimplnxCore/test/IdentifySampleBenchmarkUtilities.hpp new file mode 100644 index 0000000000..bdf43ccefa --- /dev/null +++ b/src/Plugins/SimplnxCore/test/IdentifySampleBenchmarkUtilities.hpp @@ -0,0 +1,556 @@ +#pragma once + +#include "IdentifySampleBatchTestUtilities.hpp" +#include "SimplnxCore/Filters/Algorithms/IdentifySampleCCL.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" + +#include +#include + +#ifdef SIMPLNX_ENABLE_MULTICORE +#include +#endif +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace IdentifySampleBenchmarkTest +{ +using namespace nx::core; +constexpr usize k_Dimension = 200; +constexpr usize k_XyValues = k_Dimension * k_Dimension; +constexpr uint64 k_DefaultBudget = 536870912; +constexpr usize k_RequestedThreads = 8; +using Clock = std::chrono::steady_clock; +using Json = nlohmann::json; + +/** + * @brief Parses a positive uint64 field with at most twenty decimal digits. + * @param text Complete decimal field. + * @param name Identifies the field in an error message. + * @return Parsed value. + * @throws std::invalid_argument If the field is empty, too long, zero, out of range, or contains another character. + */ +inline uint64 ParsePositiveDecimal(std::string_view text, std::string_view name) +{ + if(text.empty() || text.size() > 20 || !std::all_of(text.begin(), text.end(), [](char value) { return value >= '0' && value <= '9'; })) + { + throw std::invalid_argument(std::string(name) + " requires one to twenty decimal digits"); + } + uint64 value = 0; + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), value); + if(parsed.ec != std::errc{} || parsed.ptr != text.data() + text.size() || value == 0) + { + throw std::invalid_argument(std::string(name) + " requires a positive uint64 value"); + } + return value; +} + +/** + * @brief Reads one bounded decimal environment field without changing the environment. + * @param name Environment-variable name. + * @param fallback Value used only when the variable is absent. + * @return Parsed value or the fallback. + * @throws std::invalid_argument If the present value is not a positive uint64 decimal field. + */ +inline uint64 ReadPositiveEnvironment(const char* name, uint64 fallback) +{ + const char* raw = std::getenv(name); + if(raw == nullptr) + { + return fallback; + } + usize length = 0; + while(length <= 20 && raw[length] != '\0') + { + length++; + } + if(length > 20) + { + throw std::invalid_argument(std::string(name) + " exceeds twenty decimal digits"); + } + return ParsePositiveDecimal(std::string_view(raw, length), name); +} + +inline int64 UtcMilliseconds() +{ + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); +} + +inline uint64 ProcessId() +{ +#ifdef _WIN32 + return static_cast(GetCurrentProcessId()); +#else + return static_cast(getpid()); +#endif +} + +// These are process-wide OS requests. Other files, metadata, and device requests can contribute. +struct ProcessIo +{ + bool available = false; + uint64 error = 0; + uint64 read = 0; + uint64 write = 0; + uint64 other = 0; +}; + +inline ProcessIo ReadProcessIo() noexcept +{ + ProcessIo result; +#ifdef _WIN32 + IO_COUNTERS counters{}; + if(GetProcessIoCounters(GetCurrentProcess(), &counters) == FALSE) + { + result.error = GetLastError(); + return result; + } + result.available = true; + result.read = counters.ReadTransferCount; + result.write = counters.WriteTransferCount; + result.other = counters.OtherTransferCount; +#endif + return result; +} + +inline Json ProcessIoDelta(const ProcessIo& before, const ProcessIo& after) +{ + if(!before.available || !after.available) + { + return {{"available", false}, {"before_error", before.error}, {"after_error", after.error}, {"scope", "Windows self-process I/O requests; unavailable on other platforms"}}; + } + REQUIRE(after.read >= before.read); + REQUIRE(after.write >= before.write); + REQUIRE(after.other >= before.other); + return {{"available", true}, + {"read_bytes", after.read - before.read}, + {"write_bytes", after.write - before.write}, + {"other_bytes", after.other - before.other}, + {"scope", "current-process OS requests; includes other files; not mask-only or physical device bytes"}}; +} + +/** + * @class BudgetRestore + * @brief Restores only the current manager budget after test-owned reservations expire. + */ +class BudgetRestore +{ +public: + BudgetRestore() + : m_Original(CacheMemoryBudgetManager::instance().budgetBytes()) + { + } + ~BudgetRestore() + { + CacheMemoryBudgetManager::instance().setBudgetBytes(m_Original); + } + BudgetRestore(const BudgetRestore&) = delete; + BudgetRestore(BudgetRestore&&) = delete; + BudgetRestore& operator=(const BudgetRestore&) = delete; + BudgetRestore& operator=(BudgetRestore&&) = delete; + +private: + uint64 m_Original; +}; + +inline bool InputValue(usize x, usize y, usize z) +{ + const usize shift = x % 3; + const bool rectangle = y >= 20 + shift && y <= 179 + shift && z >= 20 && z <= 179; + return (rectangle && (y != 80 + shift || z != 90)) || (y == 2 && z == 2); +} + +inline bool ExpectedValue(usize x, usize y, usize z, bool fillHoles) +{ + const usize shift = x % 3; + return y >= 20 + shift && y <= 179 + shift && z >= 20 && z <= 179 && (fillHoles || y != 80 + shift || z != 90); +} + +template +DataStructure CreateBenchmarkFixture() +{ + DataStructure data; + auto* image = ImageGeom::Create(data, IdentifySampleBatchTest::k_ImagePath.getTargetName()); + REQUIRE(image != nullptr); + image->setDimensions({k_Dimension, k_Dimension, k_Dimension}); + image->setSpacing({1.0F, 1.0F, 1.0F}); + image->setOrigin({0.0F, 0.0F, 0.0F}); + auto* cells = AttributeMatrix::Create(data, "CellData", {k_Dimension, k_Dimension, k_Dimension}, image->getId()); + REQUIRE(cells != nullptr); + image->setCellData(*cells); + auto store = DataStoreUtilities::CreateDataStore(data, IdentifySampleBatchTest::k_MaskPath, cells->getShape(), ShapeType{1}); + REQUIRE(store != nullptr); + auto* mask = DataArray::Create(data, "Mask", store, cells->getId()); + REQUIRE(mask != nullptr); + auto xy = std::make_unique(k_XyValues); + for(usize z = 0; z < k_Dimension; z++) + { + for(usize y = 0; y < k_Dimension; y++) + { + for(usize x = 0; x < k_Dimension; x++) + { + xy[y * k_Dimension + x] = static_cast(InputValue(x, y, z)); + } + } + const auto result = store->copyFromBuffer(z * k_XyValues, nonstd::span(xy.get(), k_XyValues)); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + return data; +} + +/** + * @brief Checks every resident output value against the independent large-fixture predicate. + * @tparam T Bool or UInt8 mask type. + * @param data Filter output with the original geometry and mask. + * @param fillHoles Selects the expected filled-hole value. + */ +template +void RequireResidentOutput(const DataStructure& data, bool fillHoles) +{ + REQUIRE_NOTHROW(data.getDataRefAs(IdentifySampleBatchTest::k_ImagePath)); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + const auto& image = data.getDataRefAs(IdentifySampleBatchTest::k_ImagePath); + const auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + REQUIRE(image.getDimensions() == SizeVec3(k_Dimension, k_Dimension, k_Dimension)); + const ShapeType expectedShape{k_Dimension, k_Dimension, k_Dimension}; + REQUIRE(mask.getTupleShape() == expectedShape); + REQUIRE(mask.getComponentShape() == ShapeType{1}); + auto xy = std::make_unique(k_XyValues); + uint64 trueCount = 0; + for(usize z = 0; z < k_Dimension; z++) + { + const auto result = mask.getDataStoreRef().copyIntoBuffer(z * k_XyValues, nonstd::span(xy.get(), k_XyValues)); + SIMPLNX_RESULT_REQUIRE_VALID(result); + for(usize y = 0; y < k_Dimension; y++) + { + for(usize x = 0; x < k_Dimension; x++) + { + const T expected = static_cast(ExpectedValue(x, y, z, fillHoles)); + if(xy[y * k_Dimension + x] != expected) + { + FAIL("Resident value mismatch at " << x << ',' << y << ',' << z); + } + trueCount += static_cast(xy[y * k_Dimension + x]); + } + } + } + REQUIRE(trueCount == (fillHoles ? 5120000ULL : 5119800ULL)); + UnitTest::CheckArraysInheritTupleDims(data); +} +/** + * @struct LogicalCounters + * @brief Accumulates CCL slice observations on the filter execution thread. + * @note The selected mask identity outlives the callback. No callback runs under storage or HDF5 locks. + * @note extraAllocatedBytes, extraRequestedBytes, and extraGrantedBytes accumulate across batches. + * @note minimumBatchWidth remains uint64's maximum until the first batch-width event. + */ +struct LogicalCounters +{ + const IDataArray* selected = nullptr; + uint64 reads = 0; + uint64 writes = 0; + uint64 readSuccess = 0; + uint64 writeSuccess = 0; + uint64 readValues = 0; + uint64 writeValues = 0; + uint64 maximumSpan = 0; + uint64 active = 0; + uint64 concurrent = 0; + uint64 planeLive = 0; + uint64 xyLive = 0; + uint64 namedCarrierPeak = 0; + uint64 allocationEvents = 0; + uint64 releaseEvents = 0; + uint64 reservationMin = std::numeric_limits::max(); + uint64 reservationMax = 0; + bool invalid = false; + + uint64 extraLiveBytes = 0; + uint64 extraPeakBytes = 0; + uint64 extraAllocatedBytes = 0; + uint64 extraAllocationCount = 0; + uint64 extraReleaseCount = 0; + uint64 extraRequestCount = 0; + uint64 extraGrantCount = 0; + uint64 extraRetainCount = 0; + uint64 extraReservationReleaseCount = 0; + uint64 extraRequestedBytes = 0; + uint64 extraRequestedMaxBytes = 0; + uint64 extraGrantedBytes = 0; + uint64 extraGrantedMaxBytes = 0; + uint64 extraRetainedBytes = 0; + uint64 extraRetainedMaxBytes = 0; + uint64 batchCount = 0; + uint64 batchPlanes = 0; + uint64 minimumBatchWidth = std::numeric_limits::max(); + uint64 maximumBatchWidth = 0; + bool invalidExtraAccounting = false; + + /** + * @brief Records transfers and actual buffer lifetimes without allocating memory. + * @param context Non-null pointer to the live counters for the selected mask. + * @param mask Identifies the source of the event. + * @param event Identifies the operation or lifetime boundary. + * @param values Gives elements, bytes, or planes as defined by the event. + * @param elementBytes Converts values to bytes; admission and width events use one. + * @param success Reports transfer completion; lifetime events require true. + */ + static void Observe(void* context, const IDataArray* mask, IdentifySampleSliceEventForTesting event, usize values, usize elementBytes, bool success) noexcept + { + auto& counts = *static_cast(context); + if(mask != counts.selected) + { + return; + } + if(elementBytes == 0 || values > std::numeric_limits::max() / elementBytes) + { + counts.invalid = true; + return; + } + const uint64 bytes = static_cast(values) * elementBytes; + const uint64 reserved = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + counts.reservationMin = std::min(counts.reservationMin, reserved); + counts.reservationMax = std::max(counts.reservationMax, reserved); + switch(event) + { + case IdentifySampleSliceEventForTesting::ReadBegin: + case IdentifySampleSliceEventForTesting::WriteBegin: + counts.concurrent = std::max(counts.concurrent, ++counts.active); + counts.maximumSpan = std::max(counts.maximumSpan, static_cast(values)); + if(event == IdentifySampleSliceEventForTesting::ReadBegin) + { + counts.reads++; + counts.readValues += values; + } + else + { + counts.writes++; + counts.writeValues += values; + } + break; + case IdentifySampleSliceEventForTesting::ReadEnd: + case IdentifySampleSliceEventForTesting::WriteEnd: + if(counts.active == 0) + { + counts.invalid = true; + break; + } + counts.active--; + if(event == IdentifySampleSliceEventForTesting::ReadEnd) + { + counts.readSuccess += static_cast(success); + } + else + { + counts.writeSuccess += static_cast(success); + } + break; + case IdentifySampleSliceEventForTesting::PlaneAllocated: + case IdentifySampleSliceEventForTesting::XyAllocated: { + auto& live = event == IdentifySampleSliceEventForTesting::PlaneAllocated ? counts.planeLive : counts.xyLive; + counts.invalid = counts.invalid || live != 0; + live = bytes; + counts.namedCarrierPeak = std::max(counts.namedCarrierPeak, counts.planeLive + counts.xyLive); + counts.allocationEvents++; + break; + } + case IdentifySampleSliceEventForTesting::PlaneReleased: + case IdentifySampleSliceEventForTesting::XyReleased: { + auto& live = event == IdentifySampleSliceEventForTesting::PlaneReleased ? counts.planeLive : counts.xyLive; + counts.invalid = counts.invalid || live != bytes; + live = 0; + counts.releaseEvents++; + break; + } + case IdentifySampleSliceEventForTesting::ExtraBytesRequested: + counts.invalidExtraAccounting = counts.invalidExtraAccounting || elementBytes != 1 || !success || counts.m_ExtraPhase != ExtraPhase::None || counts.extraLiveBytes != 0; + counts.m_ExtraPhase = ExtraPhase::Requested; + counts.m_RequestedBytes = bytes; + counts.m_GrantedBytes = 0; + counts.m_ExtraOwnerObserved = false; + counts.m_BatchWidthObserved = false; + counts.extraRequestCount++; + counts.addExtraTotal(counts.extraRequestedBytes, bytes); + counts.extraRequestedMaxBytes = std::max(counts.extraRequestedMaxBytes, bytes); + break; + case IdentifySampleSliceEventForTesting::ExtraBytesGranted: + counts.invalidExtraAccounting = counts.invalidExtraAccounting || elementBytes != 1 || !success || counts.m_ExtraPhase != ExtraPhase::Requested || bytes > counts.m_RequestedBytes; + counts.m_ExtraPhase = ExtraPhase::Granted; + counts.m_GrantedBytes = bytes; + counts.extraGrantCount++; + counts.addExtraTotal(counts.extraGrantedBytes, bytes); + counts.extraGrantedMaxBytes = std::max(counts.extraGrantedMaxBytes, bytes); + break; + case IdentifySampleSliceEventForTesting::ExtraBytesRetained: + counts.invalidExtraAccounting = counts.invalidExtraAccounting || elementBytes != 1 || !success || counts.m_ExtraPhase != ExtraPhase::Granted || bytes > counts.m_GrantedBytes; + counts.invalidExtraAccounting = counts.invalidExtraAccounting || (counts.planeLive == 0 ? bytes != 0 : bytes % counts.planeLive != 0); + counts.m_ExtraPhase = ExtraPhase::Retained; + counts.extraRetainedBytes = bytes; + counts.extraRetainCount++; + counts.extraRetainedMaxBytes = std::max(counts.extraRetainedMaxBytes, bytes); + break; + case IdentifySampleSliceEventForTesting::ExtraPlanesAllocated: + counts.invalidExtraAccounting = counts.invalidExtraAccounting || !success || counts.m_ExtraPhase != ExtraPhase::Retained || counts.m_ExtraOwnerObserved || counts.m_BatchWidthObserved || + counts.extraLiveBytes != 0 || bytes == 0 || bytes != counts.extraRetainedBytes; + counts.m_ExtraOwnerObserved = true; + counts.extraLiveBytes = bytes; + counts.extraPeakBytes = std::max(counts.extraPeakBytes, bytes); + counts.addExtraTotal(counts.extraAllocatedBytes, bytes); + counts.extraAllocationCount++; + break; + case IdentifySampleSliceEventForTesting::ExtraPlanesReleased: + counts.invalidExtraAccounting = counts.invalidExtraAccounting || !success || counts.m_ExtraPhase != ExtraPhase::Retained || !counts.m_ExtraOwnerObserved || bytes == 0 || + counts.extraLiveBytes != bytes || bytes > counts.extraRetainedBytes; + counts.extraLiveBytes = 0; + counts.extraReleaseCount++; + break; + case IdentifySampleSliceEventForTesting::BatchWidth: { + const uint64 expectedWidth = counts.planeLive == 0 ? 1 : 1 + counts.extraRetainedBytes / counts.planeLive; + counts.invalidExtraAccounting = counts.invalidExtraAccounting || elementBytes != 1 || !success || counts.m_ExtraPhase != ExtraPhase::Retained || counts.m_BatchWidthObserved || values < 1 || + values > 8 || values != expectedWidth || counts.extraLiveBytes != counts.extraRetainedBytes; + counts.m_BatchWidthObserved = true; + counts.batchCount++; + counts.addExtraTotal(counts.batchPlanes, values); + counts.minimumBatchWidth = std::min(counts.minimumBatchWidth, static_cast(values)); + counts.maximumBatchWidth = std::max(counts.maximumBatchWidth, static_cast(values)); + break; + } + case IdentifySampleSliceEventForTesting::ExtraReservationReleased: + // The token returns its grant only after the extra array releases its values. + counts.invalidExtraAccounting = + counts.invalidExtraAccounting || elementBytes != 1 || !success || counts.m_ExtraPhase != ExtraPhase::Retained || counts.extraLiveBytes != 0 || bytes != counts.extraRetainedBytes; + counts.extraRetainedBytes = 0; + counts.m_ExtraPhase = ExtraPhase::None; + counts.extraReservationReleaseCount++; + break; + } + } + + /** + * @brief Reports observed counts and peaks with separate extra-plane ownership fields. + * @return Measurements for the selected mask, excluding total process memory. + */ + Json describe() const + { + return {{"read_calls", reads}, + {"write_calls", writes}, + {"successful_read_calls", readSuccess}, + {"successful_write_calls", writeSuccess}, + {"read_values", readValues}, + {"write_values", writeValues}, + {"maximum_span_values", maximumSpan}, + {"outer_concurrency", concurrent}, + {"original_plane_and_xy_requested_peak_bytes", namedCarrierPeak}, + {"carrier_allocation_events", allocationEvents}, + {"carrier_release_events", releaseEvents}, + {"working_reservation_min_bytes", reservationMin}, + {"working_reservation_max_bytes", reservationMax}, + {"extra_plane_allocation_observation", "actual extra-array ownership; excludes original carriers, CCL records, and backend memory"}, + {"extra_plane_live_bytes", extraLiveBytes}, + {"extra_plane_peak_bytes", extraPeakBytes}, + {"extra_plane_allocated_bytes", extraAllocatedBytes}, + {"extra_plane_allocation_events", extraAllocationCount}, + {"extra_plane_release_events", extraReleaseCount}, + {"extra_request_events", extraRequestCount}, + {"extra_grant_events", extraGrantCount}, + {"extra_retain_events", extraRetainCount}, + {"extra_reservation_release_events", extraReservationReleaseCount}, + {"extra_requested_bytes", extraRequestedBytes}, + {"extra_requested_max_bytes", extraRequestedMaxBytes}, + {"extra_granted_bytes", extraGrantedBytes}, + {"extra_granted_max_bytes", extraGrantedMaxBytes}, + {"extra_retained_live_bytes", extraRetainedBytes}, + {"extra_retained_max_bytes", extraRetainedMaxBytes}, + {"batch_count", batchCount}, + {"batch_planes", batchPlanes}, + {"minimum_batch_width", batchCount == 0 ? 0 : minimumBatchWidth}, + {"maximum_batch_width", maximumBatchWidth}, + {"extra_accounting_invalid", invalidExtraAccounting}}; + } + +private: + /** + * @enum ExtraPhase + * @brief Tracks admission order separately from the manager's global reservation total. + */ + enum class ExtraPhase + { + None, ///< No admission is active. + Requested, ///< The next event must report the actual grant. + Granted, ///< The next event must report the whole-plane reservation. + Retained ///< The token remains active until all extra values are released. + }; + ExtraPhase m_ExtraPhase = ExtraPhase::None; + uint64 m_RequestedBytes = 0; + uint64 m_GrantedBytes = 0; + bool m_ExtraOwnerObserved = false; + bool m_BatchWidthObserved = false; + + /** + * @brief Adds an observation without allowing a wrapped total to appear valid. + * @param total Accumulates the observed quantity. + * @param amount Gives the increment in the same units as total. + */ + void addExtraTotal(uint64& total, uint64 amount) noexcept + { + if(amount > std::numeric_limits::max() - total) + { + invalidExtraAccounting = true; + return; + } + total += amount; + } +}; + +/** + * @brief Checks actual batch widths and extra owners for the 200-cube fixture. + * @tparam T Bool or UInt8 mask element type. + * @param counts Contains observations from one successful CCL execution. + */ +template +void RequireFullWidthYzBatchObservations(const LogicalCounters& counts) +{ + constexpr uint64 expectedBatches = k_Dimension / 8; + constexpr uint64 expectedExtraBytes = 7 * k_XyValues * sizeof(T); + static_assert(k_Dimension % 8 == 0); + REQUIRE_FALSE(counts.invalidExtraAccounting); + REQUIRE(counts.extraLiveBytes == 0); + REQUIRE(counts.extraRetainedBytes == 0); + REQUIRE(counts.extraAllocationCount == expectedBatches); + REQUIRE(counts.extraReleaseCount == expectedBatches); + REQUIRE(counts.extraRequestCount == expectedBatches); + REQUIRE(counts.extraGrantCount == expectedBatches); + REQUIRE(counts.extraRetainCount == expectedBatches); + REQUIRE(counts.extraReservationReleaseCount == expectedBatches); + REQUIRE(counts.extraAllocatedBytes == expectedBatches * expectedExtraBytes); + REQUIRE(counts.extraPeakBytes == expectedExtraBytes); + REQUIRE(counts.extraRequestedBytes == expectedBatches * expectedExtraBytes); + REQUIRE(counts.extraRequestedMaxBytes == expectedExtraBytes); + REQUIRE(counts.extraGrantedBytes == expectedBatches * expectedExtraBytes); + REQUIRE(counts.extraGrantedMaxBytes == expectedExtraBytes); + REQUIRE(counts.extraRetainedMaxBytes == expectedExtraBytes); + REQUIRE(counts.extraPeakBytes <= counts.extraRetainedMaxBytes); + REQUIRE(counts.extraRetainedMaxBytes <= counts.extraGrantedMaxBytes); + REQUIRE(counts.batchCount == expectedBatches); + REQUIRE(counts.batchPlanes == k_Dimension); + REQUIRE(counts.minimumBatchWidth == 8); + REQUIRE(counts.maximumBatchWidth == 8); +} + +} // namespace IdentifySampleBenchmarkTest diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleObservationTest.cpp b/src/Plugins/SimplnxCore/test/IdentifySampleObservationTest.cpp new file mode 100644 index 0000000000..f07b29c5c8 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/IdentifySampleObservationTest.cpp @@ -0,0 +1,378 @@ +#include "IdentifySampleBenchmarkUtilities.hpp" + +#include "SimplnxCore/Filters/Algorithms/IdentifySample.hpp" +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/DataStructure/DataStore.hpp" + +#include +#include + +using namespace nx::core; + +namespace +{ +using namespace IdentifySampleBenchmarkTest; + +/** + * @enum CopyFault + * @brief Selects the first armed copy operation and its failure carrier. + */ +enum class CopyFault +{ + None, ///< Completes all mask copies normally. + ReadResult, ///< Returns an error from the first armed read. + WriteResult, ///< Returns an error from the first armed write. + ReadThrow, ///< Throws from the first armed read. + WriteThrow ///< Throws from the first armed write. +}; + +/** + * @class FaultStore + * @brief Injects a result or exception at the first actual copy after fixture initialization. + * @tparam T Bool or UInt8 resident mask type. + */ +template +class FaultStore : public DataStore +{ +public: + /** + * @brief Creates the fixed resident mask used by the first-copy controls. + */ + FaultStore() + : DataStore(ShapeType{13, 11, 19}, ShapeType{1}, std::optional{}) + { + } + + ~FaultStore() override = default; + FaultStore(const FaultStore&) = default; + FaultStore(FaultStore&&) = default; + FaultStore& operator=(const FaultStore&) = delete; + FaultStore& operator=(FaultStore&&) = default; + + CopyFault fault = CopyFault::None; + bool armed = false; + mutable uint64 reads = 0; + uint64 writes = 0; + + /** + * @brief Delegates one read or injects the armed first-read failure. + * @param start First flat value index. + * @param values Destination values. + * @return Original copy Result or the deliberate test error. + * @throws std::runtime_error For the armed first-read exception. + */ + Result<> copyIntoBuffer(usize start, nonstd::span values) const override + { + if(armed) + { + reads++; + if(reads == 1 && fault == CopyFault::ReadResult) + { + return MakeErrorResult(-77881, "Deliberate first-copy read failure in Identify Sample observer control"); + } + if(reads == 1 && fault == CopyFault::ReadThrow) + { + throw std::runtime_error("Deliberate first-copy read exception in Identify Sample observer control"); + } + } + return DataStore::copyIntoBuffer(start, values); + } + + /** + * @brief Delegates one write or injects the armed first-write failure. + * @param start First flat value index. + * @param values Source values. + * @return Original copy Result or the deliberate test error. + * @throws std::runtime_error For the armed first-write exception. + */ + Result<> copyFromBuffer(usize start, nonstd::span values) override + { + if(armed) + { + writes++; + if(writes == 1 && fault == CopyFault::WriteResult) + { + return MakeErrorResult(-77882, "Deliberate first-copy write failure in Identify Sample observer control"); + } + if(writes == 1 && fault == CopyFault::WriteThrow) + { + throw std::runtime_error("Deliberate first-copy write exception in Identify Sample observer control"); + } + } + return DataStore::copyFromBuffer(start, values); + } +}; + +/** + * @brief Checks every mask value against the independent unmodified input predicate. + * @tparam T Bool or UInt8 mask element type. + * @param data Owns the mask after its copy faults are disarmed. + */ +template +void RequireOriginal(const DataStructure& data) +{ + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + const auto& store = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath).getDataStoreRef(); + auto xy = std::make_unique(19 * 11); + for(usize z = 0; z < 13; z++) + { + const auto read = store.copyIntoBuffer(z * 19 * 11, nonstd::span(xy.get(), 19 * 11)); + SIMPLNX_RESULT_REQUIRE_VALID(read); + for(usize y = 0; y < 11; y++) + { + for(usize x = 0; x < 19; x++) + { + REQUIRE(xy[y * 19 + x] == static_cast(IdentifySampleBatchTest::InputValue(y, z, x))); + } + } + } + UnitTest::CheckArraysInheritTupleDims(data); +} + +/** + * @brief Checks exact one-plane observations with no extra working-memory headroom. + * @tparam T Bool or UInt8 mask element type. + * @param fault Selects a first-copy failure or normal completion. + * @param matchIdentity True to observe this mask; false to test identity filtering. + * @param installObserver True to install the callback; false to test the null-observer path. + */ +template +void CheckControl(CopyFault fault, bool matchIdentity, bool installObserver) +{ + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + const BudgetRestore restoreBudget; + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + // Keep first-copy controls on their exact one-plane schedule. + const auto occupied = budget.reserveWorkingMemory(budget.maximumWorkingMemoryBytes()); + REQUIRE(occupied.sizeBytes() == budget.maximumWorkingMemoryBytes()); + auto store = std::make_shared>(); + DataStructure data = IdentifySampleBatchTest::CreateFixture(2, 19, store); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + REQUIRE(mask.getDataStore()->getStoreType() == IDataStore::StoreType::InMemory); + LogicalCounters counters; + counters.selected = matchIdentity ? &mask : nullptr; + store->fault = fault; + IdentifySampleInputValues inputs{}; + inputs.FillHoles = false; + inputs.InputImageGeometryPath = IdentifySampleBatchTest::k_ImagePath; + inputs.MaskArrayPath = IdentifySampleBatchTest::k_MaskPath; + inputs.SliceBySlice = true; + inputs.SliceBySlicePlaneIndex = 2; + const std::atomic_bool cancel = false; + const IFilter::MessageHandler messages; + IdentifySampleCCL algorithm(data, messages, cancel, &inputs); + + const auto initial = SetIdentifySampleSliceObserverForTesting({}); + const auto restoreInitial = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(initial); }); + REQUIRE(initial.callback == nullptr); + { + const IdentifySampleSliceObserverForTesting installed = installObserver ? IdentifySampleSliceObserverForTesting{&counters, &LogicalCounters::Observe} : IdentifySampleSliceObserverForTesting{}; + const auto previous = SetIdentifySampleSliceObserverForTesting(installed); + const auto restore = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(previous); }); + REQUIRE(previous.callback == nullptr); + store->armed = true; + const auto disarm = MakeScopeGuard([&]() noexcept { store->armed = false; }); + if(fault == CopyFault::ReadThrow || fault == CopyFault::WriteThrow) + { + REQUIRE_THROWS_AS(algorithm(), std::runtime_error); + } + else + { + const auto result = algorithm(); + if(fault == CopyFault::None) + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + else + { + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.errors().front().code == (fault == CopyFault::ReadResult ? -77881 : -77882)); + } + } + } + const auto restored = SetIdentifySampleSliceObserverForTesting({}); + REQUIRE(restored.callback == nullptr); + REQUIRE(restored.context == nullptr); + REQUIRE_FALSE(counters.invalid); + REQUIRE(counters.active == 0); + REQUIRE(counters.planeLive == 0); + REQUIRE(counters.xyLive == 0); + if(matchIdentity && installObserver) + { + REQUIRE(counters.reads == store->reads); + REQUIRE(counters.writes == store->writes); + REQUIRE(counters.concurrent == 1); + REQUIRE(counters.allocationEvents == 2); + REQUIRE(counters.releaseEvents == 2); + REQUIRE(counters.namedCarrierPeak == (11 * 13 + 19 * 11) * sizeof(T)); + if(fault == CopyFault::None) + { + REQUIRE(counters.reads == 494); + REQUIRE(counters.writes == 247); + REQUIRE(counters.readSuccess == 494); + REQUIRE(counters.writeSuccess == 247); + } + else if(fault == CopyFault::ReadResult || fault == CopyFault::ReadThrow) + { + REQUIRE(counters.reads == 1); + REQUIRE(counters.writes == 0); + REQUIRE(counters.readSuccess == 0); + REQUIRE(counters.writeSuccess == 0); + } + else + { + REQUIRE(counters.reads == 14); + REQUIRE(counters.writes == 1); + REQUIRE(counters.readSuccess == 14); + REQUIRE(counters.writeSuccess == 0); + } + } + else + { + REQUIRE(counters.reads == 0); + REQUIRE(counters.writes == 0); + REQUIRE(counters.allocationEvents == 0); + REQUIRE(counters.releaseEvents == 0); + } + if(fault == CopyFault::None) + { + IdentifySampleBatchTest::RequireOutput(data, 2, false); + } + else + { + RequireOriginal(data); + } +} +} // namespace + +TEST_CASE("IdentifySample observation preserves results through success and copy failures", "[.IdentifySampleObservation]") +{ + UnitTest::LoadPlugins(); + const auto fault = GENERATE(CopyFault::None, CopyFault::ReadResult, CopyFault::WriteResult, CopyFault::ReadThrow, CopyFault::WriteThrow); + const bool useBool = GENERATE(false, true); + CAPTURE(static_cast(fault), useBool); + if(useBool) + { + CheckControl(fault, true, true); + } + else + { + CheckControl(fault, true, true); + } +} + +TEST_CASE("IdentifySample observation ignores null and unrelated identities", "[.IdentifySampleObservation]") +{ + UnitTest::LoadPlugins(); + const bool installObserver = GENERATE(false, true); + CAPTURE(installObserver); + CheckControl(CopyFault::None, false, installObserver); +} + +TEST_CASE("IdentifySample observation restores nested observers and isolates threads", "[.IdentifySampleObservation]") +{ + LogicalCounters outer; + LogicalCounters inner; + const auto initial = SetIdentifySampleSliceObserverForTesting({&outer, &LogicalCounters::Observe}); + const auto restoreInitial = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(initial); }); + REQUIRE(initial.callback == nullptr); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&inner, &LogicalCounters::Observe}); + const auto restorePrior = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + REQUIRE(prior.context == &outer); + REQUIRE(prior.callback == &LogicalCounters::Observe); + } + std::atomic_bool childWasNull = false; + std::thread worker([&] { + const auto prior = SetIdentifySampleSliceObserverForTesting({}); + childWasNull = prior.context == nullptr && prior.callback == nullptr; + (void)SetIdentifySampleSliceObserverForTesting(prior); + }); + worker.join(); + REQUIRE(childWasNull.load()); + const auto current = SetIdentifySampleSliceObserverForTesting({&outer, &LogicalCounters::Observe}); + REQUIRE(current.context == &outer); + REQUIRE(current.callback == &LogicalCounters::Observe); +} + +TEST_CASE("IdentifySample benchmark decimal fields reject malformed values", "[.IdentifySampleObservation]") +{ + const auto invalid = GENERATE(std::string{}, std::string{"0"}, std::string{"-1"}, std::string{"+1"}, std::string{" 1"}, std::string{"1 "}, std::string{"1x"}, std::string{"1.0"}, + std::string{"18446744073709551616"}, std::string{"111111111111111111111"}); + CAPTURE(invalid); + REQUIRE_THROWS_AS(ParsePositiveDecimal(invalid, "fixture"), std::invalid_argument); + REQUIRE(ParsePositiveDecimal("1", "fixture") == 1); + REQUIRE(ParsePositiveDecimal("4194304", "fixture") == 4194304); + REQUIRE(ParsePositiveDecimal("536870912", "fixture") == 536870912); + REQUIRE(ParsePositiveDecimal("18446744073709551615", "fixture") == std::numeric_limits::max()); +} + +TEST_CASE("IdentifySample positive headroom batches YZ transfers", "[.IdentifySampleObservation][.IdentifySampleBatchScheduleRed]") +{ + UnitTest::LoadPlugins(); + const bool useBool = GENERATE(false, true); + const bool fillHoles = GENERATE(false, true); + DYNAMIC_SECTION("Bool=" << useBool << " fill holes=" << fillHoles) + { + const auto check = [&]() { + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + DataStructure data = IdentifySampleBatchTest::CreateFixture(2); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + REQUIRE(mask.getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const BudgetRestore restoreBudget; + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + const uint64 capacity = budget.maximumWorkingMemoryBytes(); + constexpr uint64 k_ExtraBytes = 7 * 11 * 13 * sizeof(T); + auto occupied = budget.reserveWorkingMemory(capacity - k_ExtraBytes); + REQUIRE(occupied.sizeBytes() == capacity - k_ExtraBytes); + LogicalCounters counters; + counters.selected = &mask; + IdentifySampleInputValues inputs{}; + inputs.FillHoles = fillHoles; + inputs.InputImageGeometryPath = IdentifySampleBatchTest::k_ImagePath; + inputs.MaskArrayPath = IdentifySampleBatchTest::k_MaskPath; + inputs.SliceBySlice = true; + inputs.SliceBySlicePlaneIndex = 2; + const std::atomic_bool cancel = false; + const IFilter::MessageHandler messages; + IdentifySampleCCL algorithm(data, messages, cancel, &inputs); + { + const auto prior = SetIdentifySampleSliceObserverForTesting({&counters, &LogicalCounters::Observe}); + const auto restore = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(prior); }); + REQUIRE(prior.callback == nullptr); + const auto result = algorithm(); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == occupied.sizeBytes()); + REQUIRE_FALSE(counters.invalid); + REQUIRE(counters.active == 0); + REQUIRE(counters.planeLive == 0); + REQUIRE(counters.xyLive == 0); + REQUIRE(counters.concurrent == 1); + REQUIRE(counters.maximumSpan == 19 * 11); + IdentifySampleBatchTest::RequireOutput(data, 2, fillHoles); + // Nineteen columns require three batches of at most eight columns. + CHECK(counters.reads == 2 * 3 * 13); + CHECK(counters.writes == 3 * 13); + CHECK(counters.readSuccess == counters.reads); + CHECK(counters.writeSuccess == counters.writes); + }; + if(useBool) + { + check.template operator()(); + } + else + { + check.template operator()(); + } + } +} diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleTest.cpp b/src/Plugins/SimplnxCore/test/IdentifySampleTest.cpp index ba4e199ff1..560bdf1145 100644 --- a/src/Plugins/SimplnxCore/test/IdentifySampleTest.cpp +++ b/src/Plugins/SimplnxCore/test/IdentifySampleTest.cpp @@ -1,10 +1,15 @@ +#include "IdentifySampleBatchControlUtilities.hpp" + +#include "IdentifySampleBatchTestUtilities.hpp" #include "SimplnxCore/Filters/IdentifySampleFilter.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" +#include "simplnx/Common/ScopeGuard.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/Parameters/ChoicesParameter.hpp" @@ -12,12 +17,16 @@ #include "simplnx/Pipeline/PipelineFilter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/Utilities/AlgorithmDispatch.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include +#include #include +#include #include +#include #include using namespace nx::core; @@ -380,3 +389,331 @@ TEST_CASE("SimplnxCore::IdentifySampleFilter: Geometry cell tuple count", "[Simp REQUIRE(errorRef.message.find("'Image' has 12 cells") != std::string::npos); } } + +namespace +{ +/** + * @class IdentifySampleBudgetGuard + * @brief Restores the live manager budget without changing the saved preference. + * @note The caller must release all tokens before this guard is destroyed. + */ +class IdentifySampleBudgetGuard +{ +public: + /** @brief Captures the current manager budget. */ + IdentifySampleBudgetGuard() + : m_PreviousBudget(CacheMemoryBudgetManager::instance().budgetBytes()) + { + } + + /** @brief Restores the captured budget after the test releases its tokens. */ + ~IdentifySampleBudgetGuard() + { + CacheMemoryBudgetManager::instance().setBudgetBytes(m_PreviousBudget); + } + + IdentifySampleBudgetGuard(const IdentifySampleBudgetGuard&) = delete; + IdentifySampleBudgetGuard(IdentifySampleBudgetGuard&&) = delete; + IdentifySampleBudgetGuard& operator=(const IdentifySampleBudgetGuard&) = delete; + IdentifySampleBudgetGuard& operator=(IdentifySampleBudgetGuard&&) = delete; + +private: + uint64 m_PreviousBudget; +}; + +/** + * @struct IdentifySampleTransferCounts + * @brief Records logical bulk calls and live working reservations during target execution. + */ +struct IdentifySampleTransferCounts +{ + usize readCalls = 0; + usize writeCalls = 0; + usize successfulReads = 0; + usize successfulWrites = 0; + usize requestedReadValues = 0; + usize requestedWriteValues = 0; + usize maximumSpan = 0; + usize activeCalls = 0; + usize maximumConcurrentCalls = 0; + uint64 minimumReservedBytes = std::numeric_limits::max(); + uint64 maximumReservedBytes = 0; +}; + +/** + * @class IdentifySampleReportingStore + * @brief Observes real resident bulk transfers only while a witnessed filter call is active. + * @tparam T Bool or UInt8 mask element type. + * @note Counters measure logical calls. They do not measure disk traffic or backend worker concurrency. + */ +template +class IdentifySampleReportingStore : public DataStore +{ +public: + /** + * @brief Creates a real resident store with observation disabled. + * @param tupleShape Mask tuple dimensions in Z, Y, X order. + */ + explicit IdentifySampleReportingStore(const ShapeType& tupleShape) + : DataStore(tupleShape, ShapeType{1}, std::optional{}) + { + } + + ~IdentifySampleReportingStore() override = default; + IdentifySampleReportingStore(const IdentifySampleReportingStore&) = delete; + IdentifySampleReportingStore(IdentifySampleReportingStore&&) = delete; + IdentifySampleReportingStore& operator=(const IdentifySampleReportingStore&) = delete; + IdentifySampleReportingStore& operator=(IdentifySampleReportingStore&&) = delete; + + /** + * @brief Observes one bulk read and delegates the transfer to the resident store exactly once. + * @param startIndex First flat source value index. + * @param buffer Receives the source values. + * @return The resident store transfer result. + */ + Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + if(!m_Armed) + { + return DataStore::copyIntoBuffer(startIndex, buffer); + } + beginTransfer(true, buffer.size()); + const auto completion = MakeScopeGuard([this]() noexcept { endTransfer(); }); + auto result = DataStore::copyIntoBuffer(startIndex, buffer); + if(result.valid()) + { + const std::lock_guard lock(m_CountMutex); + m_Counts.successfulReads++; + } + return result; + } + + /** + * @brief Observes one bulk write and delegates the transfer to the resident store exactly once. + * @param startIndex First flat destination value index. + * @param buffer Supplies the new values. + * @return The resident store transfer result. + */ + Result<> copyFromBuffer(usize startIndex, nonstd::span buffer) override + { + if(!m_Armed) + { + return DataStore::copyFromBuffer(startIndex, buffer); + } + beginTransfer(false, buffer.size()); + const auto completion = MakeScopeGuard([this]() noexcept { endTransfer(); }); + auto result = DataStore::copyFromBuffer(startIndex, buffer); + if(result.valid()) + { + const std::lock_guard lock(m_CountMutex); + m_Counts.successfulWrites++; + } + return result; + } + + /** + * @brief Enables or disables observation between transfers. + * @param armed True to count subsequent calls. + * @pre No transfer is active. + */ + void setArmed(bool armed) noexcept + { + m_Armed = armed; + } + + IdentifySampleTransferCounts counts() const + { + const std::lock_guard lock(m_CountMutex); + return m_Counts; + } + +private: + void beginTransfer(bool isRead, usize values) const + { + const uint64 reserved = CacheMemoryBudgetManager::instance().reservedWorkingMemoryBytes(); + const std::lock_guard lock(m_CountMutex); + if(isRead) + { + m_Counts.readCalls++; + m_Counts.requestedReadValues += values; + } + else + { + m_Counts.writeCalls++; + m_Counts.requestedWriteValues += values; + } + m_Counts.maximumSpan = std::max(m_Counts.maximumSpan, values); + m_Counts.maximumConcurrentCalls = std::max(m_Counts.maximumConcurrentCalls, ++m_Counts.activeCalls); + m_Counts.minimumReservedBytes = std::min(m_Counts.minimumReservedBytes, reserved); + m_Counts.maximumReservedBytes = std::max(m_Counts.maximumReservedBytes, reserved); + } + + void endTransfer() const noexcept + { + const std::lock_guard lock(m_CountMutex); + m_Counts.activeCalls--; + } + + bool m_Armed = false; + mutable std::mutex m_CountMutex; + mutable IdentifySampleTransferCounts m_Counts; +}; + +/** + * @brief Verifies exact output and the CCL one-plane schedule with no available working reservation. + * @tparam T Bool or UInt8 mask element type. + * @param scope Selects and witnesses the resident algorithm scenario. + * @param fillHoles Selects hole filling. + */ +template +void CheckIdentifySampleWithoutWorkingHeadroom(UnitTest::AlgorithmTestScope& scope, bool fillHoles) +{ + constexpr usize k_X = 19; + constexpr usize k_Y = 11; + constexpr usize k_Z = 13; + + using IdentifySampleReportingStoreType = IdentifySampleReportingStore; + using DataArrayType = DataArray; + + auto store = std::make_shared(ShapeType{k_Z, k_Y, k_X}); + DataStructure dataStructure = IdentifySampleBatchTest::CreateFixture(2, k_X, store); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(IdentifySampleBatchTest::k_MaskPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(IdentifySampleBatchTest::k_MaskPath)); + + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const uint64 originalBudget = budget.budgetBytes(); + { + const IdentifySampleBudgetGuard budgetGuard; + REQUIRE_FALSE(budget.setBudgetBytes(4ULL * 1024 * 1024)); + REQUIRE(budget.budgetBytes() == 4ULL * 1024 * 1024); + const uint64 capacity = budget.maximumWorkingMemoryBytes(); + REQUIRE(capacity == 1024ULL * 1024); + const auto occupied = budget.reserveWorkingMemory(capacity); + REQUIRE(occupied.sizeBytes() == capacity); + REQUIRE(budget.reservedWorkingMemoryBytes() == capacity); + const auto refused = budget.reserveWorkingMemory(1); + REQUIRE(refused.sizeBytes() == 0); + + IdentifySampleFilter filter; + const auto args = IdentifySampleBatchTest::ArgumentsFor(2, fillHoles); + const auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + { + store->setArmed(true); + const auto disarm = MakeScopeGuard([&store]() noexcept { store->setArmed(false); }); + const auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + } + + if(scope.scenario() == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore) + { + // With no headroom, CCL retains the one-plane transfer schedule and full XY spans. + const auto counts = store->counts(); + REQUIRE(counts.readCalls == 2 * k_X * k_Z); + REQUIRE(counts.writeCalls == k_X * k_Z); + REQUIRE(counts.successfulReads == counts.readCalls); + REQUIRE(counts.successfulWrites == counts.writeCalls); + REQUIRE(counts.requestedReadValues == counts.readCalls * k_X * k_Y); + REQUIRE(counts.requestedWriteValues == counts.writeCalls * k_X * k_Y); + REQUIRE(counts.maximumSpan == k_X * k_Y); + REQUIRE(counts.maximumConcurrentCalls == 1); + REQUIRE(counts.activeCalls == 0); + REQUIRE(counts.minimumReservedBytes == capacity); + REQUIRE(counts.maximumReservedBytes == capacity); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == capacity); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + REQUIRE(budget.budgetBytes() == originalBudget); + IdentifySampleBatchTest::RequireOutput(dataStructure, 2, fillHoles, k_X); +} +} // namespace + +TEST_CASE("SimplnxCore::IdentifySampleFilter: orthogonal literal masks", "[SimplnxCore][IdentifySampleFilter][.IdentifySampleBatchCorrectness]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const auto plane = GENERATE(ChoicesParameter::ValueType{0}, ChoicesParameter::ValueType{1}, ChoicesParameter::ValueType{2}); + const bool fillHoles = GENERATE(false, true); + const bool useBool = GENERATE(false, true); + CAPTURE(scenario, plane, fillHoles, useBool); + const IdentifySampleBatchTest::ScopedPreferenceRestore preferenceRestore; + UnitTest::AlgorithmTestScope scope(scenario); + if(useBool) + { + auto dataStructure = IdentifySampleBatchTest::CreateFixture(plane); + IdentifySampleBatchTest::ExecuteAndRequire(scope, dataStructure, plane, fillHoles); + } + else + { + auto dataStructure = IdentifySampleBatchTest::CreateFixture(plane); + IdentifySampleBatchTest::ExecuteAndRequire(scope, dataStructure, plane, fillHoles); + } +} + +TEST_CASE("SimplnxCore::IdentifySampleFilter: YZ literal tail masks", "[SimplnxCore][IdentifySampleFilter][.IdentifySampleBatchCorrectness]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const usize fixedCount = GENERATE(usize{1}, usize{7}, usize{8}, usize{9}, usize{16}, usize{17}, usize{19}); + const bool fillHoles = GENERATE(false, true); + const bool useBool = GENERATE(false, true); + CAPTURE(scenario, fixedCount, fillHoles, useBool); + const IdentifySampleBatchTest::ScopedPreferenceRestore preferenceRestore; + UnitTest::AlgorithmTestScope scope(scenario); + if(useBool) + { + auto dataStructure = IdentifySampleBatchTest::CreateFixture(2, fixedCount); + IdentifySampleBatchControlTest::CheckGrant(scope, dataStructure, 8, 0, fillHoles, fixedCount, scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); + } + else + { + auto dataStructure = IdentifySampleBatchTest::CreateFixture(2, fixedCount); + IdentifySampleBatchControlTest::CheckGrant(scope, dataStructure, 8, 0, fillHoles, fixedCount, scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); + } +} + +TEST_CASE("SimplnxCore::IdentifySampleFilter: YZ without working headroom", "[SimplnxCore][IdentifySampleFilter][.IdentifySampleBatchCorrectness]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool fillHoles = GENERATE(false, true); + const bool useBool = GENERATE(false, true); + CAPTURE(scenario, fillHoles, useBool); + const IdentifySampleBatchTest::ScopedPreferenceRestore preferenceRestore; + UnitTest::AlgorithmTestScope scope(scenario); + if(useBool) + { + CheckIdentifySampleWithoutWorkingHeadroom(scope, fillHoles); + } + else + { + CheckIdentifySampleWithoutWorkingHeadroom(scope, fillHoles); + } +} + +TEST_CASE("SimplnxCore::IdentifySampleFilter: YZ real reservation widths", "[SimplnxCore][IdentifySampleFilter][.IdentifySampleBatchCorrectness]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const usize width = GENERATE(usize{1}, usize{2}, usize{3}, usize{8}); + const usize partial = GENERATE(usize{0}, usize{17}); + const bool fillHoles = GENERATE(false, true); + const bool useBool = GENERATE(false, true); + DYNAMIC_SECTION("scenario=" << scenario << " width=" << width << " remainder=" << partial << " Bool=" << useBool << " fill=" << fillHoles) + { + const IdentifySampleBatchTest::ScopedPreferenceRestore restorePreferences; + UnitTest::AlgorithmTestScope scope(scenario); + if(useBool) + { + auto data = IdentifySampleBatchTest::CreateFixture(2); + IdentifySampleBatchControlTest::CheckGrant(scope, data, width, partial, fillHoles, 19, scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); + } + else + { + auto data = IdentifySampleBatchTest::CreateFixture(2); + IdentifySampleBatchControlTest::CheckGrant(scope, data, width, partial, fillHoles, 19, scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); + } + } +} diff --git a/src/Plugins/SimplnxCore/test/IdentifySampleYzResidentBenchmark.cpp b/src/Plugins/SimplnxCore/test/IdentifySampleYzResidentBenchmark.cpp new file mode 100644 index 0000000000..7a3f96d991 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/IdentifySampleYzResidentBenchmark.cpp @@ -0,0 +1,163 @@ +#include "IdentifySampleBenchmarkUtilities.hpp" + +#include "simplnx/Common/ScopeGuard.hpp" + +#include +#include + +using namespace nx::core; + +namespace +{ +using namespace IdentifySampleBenchmarkTest; + +/** + * @brief Measures one supported resident algorithm scenario on the same large predicate as the actual-OOC fixture. + * @tparam T Bool or UInt8 mask element type. + * @param fillHoles Selects hole filling. + * @param requestedScenario Selects BFS or CCL while the store remains resident. + */ +template +void MeasureResident(bool fillHoles, UnitTest::AlgorithmTestScenario requestedScenario) +{ + UnitTest::LoadPlugins(); + const auto scenarios = UnitTest::SelectAlgorithmTestScenariosForInMemoryStores(); + REQUIRE(std::find(scenarios.begin(), scenarios.end(), requestedScenario) != scenarios.end()); + const IdentifySampleBatchTest::ScopedPreferenceRestore preferenceRestore; + UnitTest::AlgorithmTestScope scope(requestedScenario); + const uint64 requestedBudget = ReadPositiveEnvironment("SIMPLNX_IDENTIFY_BENCHMARK_CACHE_BYTES", k_DefaultBudget); + const uint64 repetition = ReadPositiveEnvironment("SIMPLNX_IDENTIFY_BENCHMARK_REPETITION", 1); + auto& budget = CacheMemoryBudgetManager::instance(); + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + const uint64 originalBudget = budget.budgetBytes(); + { + const BudgetRestore restoreBudget; + const bool clamped = budget.setBudgetBytes(requestedBudget); +#ifdef SIMPLNX_ENABLE_MULTICORE + const tbb::global_control threadLimit(tbb::global_control::max_allowed_parallelism, k_RequestedThreads); + const usize effectiveThreads = tbb::global_control::active_value(tbb::global_control::max_allowed_parallelism); +#else + const usize effectiveThreads = 1; +#endif + DataStructure data = CreateBenchmarkFixture(); + REQUIRE_NOTHROW(data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath)); + auto& mask = data.getDataRefAs>(IdentifySampleBatchTest::k_MaskPath); + scope.requireExpectedStore(mask); + REQUIRE(dynamic_cast*>(mask.getDataStore()) != nullptr); + IdentifySampleFilter filter; + const auto args = IdentifySampleBatchTest::ArgumentsFor(2, fillHoles); + const auto preflight = filter.preflight(data, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + const bool useCcl = requestedScenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore; + Json metadata = {{"version", 1}, + {"phase", "after"}, + {"algorithm", useCcl ? "CCL" : "BFS"}, + {"store", "resident"}, + {"type", std::is_same_v ? "bool" : "uint8"}, + {"fill_holes", fillHoles}, + {"repetition", repetition}, + {"pid", ProcessId()}, + {"requested_cache_bytes", requestedBudget}, + {"effective_cache_bytes", budget.budgetBytes()}, + {"cache_budget_clamped", clamped}, + {"working_capacity_bytes", budget.maximumWorkingMemoryBytes()}, + {"requested_tbb_thread_cap", k_RequestedThreads}, + {"effective_tbb_thread_cap", effectiveThreads}}; + fmt::print("P2_IDENTIFY_RESIDENT_SETUP {}\n", metadata.dump()); + REQUIRE(std::fflush(stdout) == 0); + LogicalCounters logical; + logical.selected = &mask; + Clock::time_point start; + Clock::time_point end; + int64 utcStart = 0; + int64 utcEnd = 0; + ProcessIo ioBefore; + ProcessIo ioAfter; + { + const auto previous = SetIdentifySampleSliceObserverForTesting({&logical, &LogicalCounters::Observe}); + const auto restore = MakeScopeGuard([&]() noexcept { (void)SetIdentifySampleSliceObserverForTesting(previous); }); + REQUIRE(previous.callback == nullptr); + const auto result = scope.execute([&] { + ioBefore = ReadProcessIo(); + utcStart = UtcMilliseconds(); + start = Clock::now(); + auto executed = filter.execute(data, args); + end = Clock::now(); + utcEnd = UtcMilliseconds(); + ioAfter = ReadProcessIo(); + return executed; + }); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + } + REQUIRE_FALSE(logical.invalid); + REQUIRE(logical.active == 0); + REQUIRE(logical.planeLive == 0); + REQUIRE(logical.xyLive == 0); + if(useCcl) + { + REQUIRE(logical.reads == 10000); + REQUIRE(logical.writes == 5000); + REQUIRE(logical.readSuccess == logical.reads); + REQUIRE(logical.writeSuccess == logical.writes); + REQUIRE(logical.maximumSpan == k_XyValues); + REQUIRE(logical.concurrent == 1); + REQUIRE(logical.allocationEvents == 2); + REQUIRE(logical.releaseEvents == 2); + REQUIRE(logical.namedCarrierPeak == 2 * k_XyValues * sizeof(T)); + RequireFullWidthYzBatchObservations(logical); + metadata["logical_ccl"] = logical.describe(); + } + else + { + REQUIRE(logical.reads == 0); + REQUIRE(logical.writes == 0); + REQUIRE(logical.allocationEvents == 0); + metadata["logical_ccl"] = "not applicable; selected BFS does not use the CCL observer"; + } + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + scope.requireExpectedStore(mask); + RequireResidentOutput(data, fillHoles); + metadata["execute_ms"] = std::chrono::duration(end - start).count(); + metadata["utc_epoch_ms"] = {{"execute_start", utcStart}, {"execute_end", utcEnd}}; + metadata["process_io"] = ProcessIoDelta(ioBefore, ioAfter); + metadata["persistence"] = "resident mask; no separate mask-file flush"; + metadata["correctness"] = "all 8000000 values match the independent predicate"; + fmt::print("P2_IDENTIFY_RESIDENT_MEASUREMENT {}\n", metadata.dump()); + } + REQUIRE(budget.reservedWorkingMemoryBytes() == 0); + REQUIRE(budget.budgetBytes() == originalBudget); +} +} // namespace + +TEST_CASE("IdentifySample YZ resident BFS bool noholes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(false, UnitTest::AlgorithmTestScenario::InCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident BFS bool holes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(true, UnitTest::AlgorithmTestScenario::InCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident BFS uint8 noholes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(false, UnitTest::AlgorithmTestScenario::InCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident BFS uint8 holes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(true, UnitTest::AlgorithmTestScenario::InCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident CCL bool noholes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(false, UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident CCL bool holes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(true, UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident CCL uint8 noholes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(false, UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); +} +TEST_CASE("IdentifySample YZ resident CCL uint8 holes", "[.IdentifySampleYzResidentBenchmark]") +{ + MeasureResident(true, UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore); +} diff --git a/src/Plugins/SimplnxCore/test/LaplacianSmoothingNumericalTest.cpp b/src/Plugins/SimplnxCore/test/LaplacianSmoothingNumericalTest.cpp new file mode 100644 index 0000000000..58ad466069 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/LaplacianSmoothingNumericalTest.cpp @@ -0,0 +1,371 @@ +#include "SimplnxCore/Filters/LaplacianSmoothingFilter.hpp" + +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/Geometry/EdgeGeom.hpp" +#include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/DataStructure/IDataArray.hpp" +#include "simplnx/UnitTest/AlgorithmTestScope.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" + +#include + +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +// The four-vertex cache fits without a temporary record-store provider. +class SmoothingBudgetScope +{ +public: + SmoothingBudgetScope() + : m_OriginalBudget(CacheMemoryBudgetManager::instance().budgetBytes()) + { + CacheMemoryBudgetManager::instance().setBudgetBytes(1 << 20); + } + ~SmoothingBudgetScope() + { + CacheMemoryBudgetManager::instance().setBudgetBytes(m_OriginalBudget); + } + SmoothingBudgetScope(const SmoothingBudgetScope&) = delete; + SmoothingBudgetScope(SmoothingBudgetScope&&) = delete; + SmoothingBudgetScope& operator=(const SmoothingBudgetScope&) = delete; + SmoothingBudgetScope& operator=(SmoothingBudgetScope&&) = delete; + +private: + uint64 m_OriginalBudget; +}; + +/** @brief Stores all three coordinates of the four analytic vertices. */ +using Coordinates = std::array; +/** @brief Stores one classification for each analytic vertex. */ +using NodeTypes = std::array; + +/** @brief Identifies the graph geometry selected by the filter. */ +const DataPath k_GeometryPath{{"Mesh"}}; +/** @brief Identifies the coordinate array modified in place. */ +const DataPath k_VerticesPath{{"Mesh", "Vertices"}}; +/** @brief Identifies the unchanged vertex classifications. */ +const DataPath k_NodeTypesPath{{"Mesh", "Vertex Data", "Node Types"}}; +/** @brief Identifies the supplied faces or edges that must survive smoothing. */ +const DataPath k_ConnectivityPath{{"Mesh", "Connectivity"}}; +/** @brief Supplies a tetrahedron with centroid (1/2, 1/2, 1/2). */ +constexpr Coordinates k_InputCoordinates{0.0F, 0.0F, 0.0F, 2.0F, 0.0F, 0.0F, 0.0F, 2.0F, 0.0F, 0.0F, 0.0F, 2.0F}; +/** @brief Supplies the four faces whose unique edges form the complete four-vertex graph. */ +constexpr std::array k_Faces{0, 2, 1, 0, 1, 3, 0, 3, 2, 1, 2, 3}; +/** @brief Supplies the same complete graph directly as six edges. */ +constexpr std::array k_Edges{0, 1, 0, 2, 0, 3, 1, 2, 1, 3, 2, 3}; + +/** + * @enum GeometryKind + * @brief Selects generated or supplied edge connectivity for the same graph. + */ +enum class GeometryKind +{ + Triangle, ///< Generates edges from four triangular faces. + Edge ///< Uses six supplied edges. +}; + +/** + * @struct NodeTypeCase + * @brief Pairs one node classification with independently derived one-pass coordinates. + */ +struct NodeTypeCase +{ + std::string_view name; + int8 nodeType; + Coordinates expected; +}; + +// For K4, a uniform pass scales each displacement from the centroid by 1 - 4*lambda/3. +// These explicit coordinates use that closed form, not the production edge traversal. +constexpr std::array k_NodeTypeCases{{ + {"Unused", NodeType::Unused, k_InputCoordinates}, + {"Default", NodeType::Default, {1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 6.0F, 3.0F / 2.0F, 1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 6.0F, 3.0F / 2.0F, 1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 6.0F, 3.0F / 2.0F}}, + {"TriplePoint", NodeType::TriplePoint, {1.0F / 3.0F, 1.0F / 3.0F, 1.0F / 3.0F, 1.0F, 1.0F / 3.0F, 1.0F / 3.0F, 1.0F / 3.0F, 1.0F, 1.0F / 3.0F, 1.0F / 3.0F, 1.0F / 3.0F, 1.0F}}, + {"QuadPoint", NodeType::QuadPoint, {0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F, 0.5F}}, + {"SurfaceDefault", + NodeType::SurfaceDefault, + {1.0F / 12.0F, 1.0F / 12.0F, 1.0F / 12.0F, 7.0F / 4.0F, 1.0F / 12.0F, 1.0F / 12.0F, 1.0F / 12.0F, 7.0F / 4.0F, 1.0F / 12.0F, 1.0F / 12.0F, 1.0F / 12.0F, 7.0F / 4.0F}}, + {"SurfaceTriplePoint", NodeType::SurfaceTriplePoint, {0.25F, 0.25F, 0.25F, 1.25F, 0.25F, 0.25F, 0.25F, 1.25F, 0.25F, 0.25F, 0.25F, 1.25F}}, + {"SurfaceQuadPoint", + NodeType::SurfaceQuadPoint, + {5.0F / 12.0F, 5.0F / 12.0F, 5.0F / 12.0F, 0.75F, 5.0F / 12.0F, 5.0F / 12.0F, 5.0F / 12.0F, 0.75F, 5.0F / 12.0F, 5.0F / 12.0F, 5.0F / 12.0F, 0.75F}}, +}}; + +/** + * @struct IterationCase + * @brief Supplies an independent oracle for repeated passes or the optional Taubin pass. + */ +struct IterationCase +{ + std::string_view name; + int32 iterations; + bool useTaubin; + Coordinates expected; +}; + +// Two lambda=1/4 passes have centroid factor 4/9. One Taubin pair with a mu factor of -1 has a centroid factor of 8/9. +constexpr std::array k_IterationCases{{ + {"zero plain iterations", 0, false, k_InputCoordinates}, + {"zero Taubin iterations", 0, true, k_InputCoordinates}, + {"two plain iterations", + 2, + false, + {5.0F / 18.0F, 5.0F / 18.0F, 5.0F / 18.0F, 7.0F / 6.0F, 5.0F / 18.0F, 5.0F / 18.0F, 5.0F / 18.0F, 7.0F / 6.0F, 5.0F / 18.0F, 5.0F / 18.0F, 5.0F / 18.0F, 7.0F / 6.0F}}, + {"one Taubin iteration", + 1, + true, + {1.0F / 18.0F, 1.0F / 18.0F, 1.0F / 18.0F, 11.0F / 6.0F, 1.0F / 18.0F, 1.0F / 18.0F, 1.0F / 18.0F, 11.0F / 6.0F, 1.0F / 18.0F, 1.0F / 18.0F, 1.0F / 18.0F, 11.0F / 6.0F}}, +}}; + +/** + * @brief Creates one small input array through the destination's storage resolver. + * @tparam T Specifies the scalar type. + * @tparam N Specifies the literal input value count. + * @param dataStructure Owns the array and supplies storage policy. + * @param path Names the new array. + * @param parentId Identifies its existing parent. + * @param tuples Supplies the tuple count. + * @param components Supplies values per tuple. + * @param values Supplies every initial value. + * @return The inserted array. + */ +template +DataArray* CreateInputArray(DataStructure& dataStructure, const DataPath& path, DataObject::IdType parentId, usize tuples, usize components, const std::array& values) +{ + REQUIRE(values.size() == tuples * components); + auto store = DataStoreUtilities::CreateDataStore(dataStructure, path, {tuples}, {components}); + REQUIRE(store != nullptr); + auto* arrayPtr = DataArray::Create(dataStructure, path.getTargetName(), store, parentId); + REQUIRE(arrayPtr != nullptr); + auto writeResult = store->copyFromBuffer(0, {values.data(), values.size()}); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + return arrayPtr; +} + +/** + * @brief Builds the same four-vertex graph from triangular faces or explicit edges. + * @param dataStructure Receives the complete fixture. + * @param kind Selects the connectivity representation. + * @param nodeTypes Supplies the four vertex classifications. + */ +void BuildGraph(DataStructure& dataStructure, GeometryKind kind, const NodeTypes& nodeTypes) +{ + INodeGeometry1D* geometryPtr = nullptr; + if(kind == GeometryKind::Triangle) + { + geometryPtr = TriangleGeom::Create(dataStructure, "Mesh"); + } + else + { + geometryPtr = EdgeGeom::Create(dataStructure, "Mesh"); + } + REQUIRE(geometryPtr != nullptr); + auto* verticesPtr = CreateInputArray(dataStructure, k_VerticesPath, geometryPtr->getId(), 4, 3, k_InputCoordinates); + geometryPtr->setVertices(*verticesPtr); + auto* vertexDataPtr = AttributeMatrix::Create(dataStructure, "Vertex Data", {4}, geometryPtr->getId()); + REQUIRE(vertexDataPtr != nullptr); + geometryPtr->setVertexAttributeMatrix(*vertexDataPtr); + CreateInputArray(dataStructure, k_NodeTypesPath, vertexDataPtr->getId(), 4, 1, nodeTypes); + if(kind == GeometryKind::Triangle) + { + auto* facesPtr = CreateInputArray(dataStructure, k_ConnectivityPath, geometryPtr->getId(), 4, 3, k_Faces); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_GeometryPath)); + auto& triangle = dataStructure.getDataRefAs(k_GeometryPath); + triangle.setFaceList(*facesPtr); + REQUIRE(triangle.getEdges() == nullptr); + } + else + { + auto* edgesPtr = CreateInputArray(dataStructure, k_ConnectivityPath, geometryPtr->getId(), 6, 2, k_Edges); + geometryPtr->setEdgeList(*edgesPtr); + } +} + +/** + * @brief Requires resident storage for each named smoothing input. + * @param dataStructure Supplies the fixture before or after execution. + */ +void RequireResidentInputs(const DataStructure& dataStructure) +{ + for(const auto& path : {k_VerticesPath, k_NodeTypesPath, k_ConnectivityPath}) + { + CAPTURE(path.toString()); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(path)); + const auto& array = dataStructure.getDataRefAs(path); + REQUIRE(array.getIDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + } +} + +/** + * @brief Runs the public filter with distinct weights for all supported node classes. + * @param scope Proves the selected algorithm and resident stores. + * @param dataStructure Contains the mutable fixture. + * @param iterations Supplies the number of smoothing iterations. + * @param useTaubin Enables the second pass with mu factor -1. + */ +void ExecuteSmoothing(UnitTest::AlgorithmTestScope& scope, DataStructure& dataStructure, int32 iterations, bool useTaubin) +{ + LaplacianSmoothingFilter filter; + Arguments args; + args.insertOrAssign(LaplacianSmoothingFilter::k_IterationSteps_Key, std::make_any(iterations)); + args.insertOrAssign(LaplacianSmoothingFilter::k_Lambda_Key, std::make_any(0.25F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_UseTaubinSmoothing_Key, std::make_any(useTaubin)); + args.insertOrAssign(LaplacianSmoothingFilter::k_MuFactor_Key, std::make_any(-1.0F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_TripleLineLambda_Key, std::make_any(0.5F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_QuadPointLambda_Key, std::make_any(0.75F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfacePointLambda_Key, std::make_any(0.125F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfaceTripleLineLambda_Key, std::make_any(0.375F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfaceQuadPointLambda_Key, std::make_any(0.625F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfaceMeshNodeTypeArrayPath_Key, std::make_any(k_NodeTypesPath)); + args.insertOrAssign(LaplacianSmoothingFilter::k_TriangleGeometryDataPath_Key, std::make_any(k_GeometryPath)); + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); +} + +/** + * @brief Checks every output coordinate and the unchanged graph inputs. + * @param dataStructure Supplies the completed fixture. + * @param kind Selects the original connectivity and derived-edge cleanup contract. + * @param nodeTypes Supplies the original classifications. + * @param expected Supplies all twelve independently derived coordinates. + * @param exact Requires exact coordinate equality for zero iterations. + */ +void CheckResult(const DataStructure& dataStructure, GeometryKind kind, const NodeTypes& nodeTypes, const Coordinates& expected, bool exact = false) +{ + RequireResidentInputs(dataStructure); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_VerticesPath)); + const auto& vertices = dataStructure.getDataRefAs(k_VerticesPath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_NodeTypesPath)); + const auto& nodes = dataStructure.getDataRefAs(k_NodeTypesPath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_ConnectivityPath)); + const auto& connectivity = dataStructure.getDataRefAs(k_ConnectivityPath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_GeometryPath)); + const auto& geometry = dataStructure.getDataRefAs(k_GeometryPath); + CHECK(geometry.getVertices() == &vertices); + REQUIRE(vertices.getTupleShape() == ShapeType{4}); + REQUIRE(vertices.getComponentShape() == ShapeType{3}); + REQUIRE(nodes.getTupleShape() == ShapeType{4}); + REQUIRE(nodes.getComponentShape() == ShapeType{1}); + for(usize vertexIdx = 0; vertexIdx < 4; ++vertexIdx) + { + CAPTURE(vertexIdx); + CHECK(nodes.getValue(vertexIdx) == nodeTypes[vertexIdx]); + for(usize compIdx = 0; compIdx < 3; ++compIdx) + { + const usize valueIdx = vertexIdx * 3 + compIdx; + const float32 actual = vertices.getValue(valueIdx); + CAPTURE(compIdx, actual, expected[valueIdx]); + if(exact) + { + CHECK(actual == expected[valueIdx]); + } + else + { + CHECK(std::abs(actual - expected[valueIdx]) <= 1.0e-6F); + } + } + } + REQUIRE(connectivity.getTupleShape() == ShapeType{kind == GeometryKind::Triangle ? usize{4} : usize{6}}); + REQUIRE(connectivity.getComponentShape() == ShapeType{kind == GeometryKind::Triangle ? usize{3} : usize{2}}); + const auto& expectedConnectivity = kind == GeometryKind::Triangle ? k_Faces : k_Edges; + for(usize valueIdx = 0; valueIdx < expectedConnectivity.size(); ++valueIdx) + { + CAPTURE(valueIdx); + CHECK(connectivity.getValue(valueIdx) == expectedConnectivity[valueIdx]); + } + if(kind == GeometryKind::Triangle) + { + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_GeometryPath)); + const auto& triangle = dataStructure.getDataRefAs(k_GeometryPath); + CHECK(triangle.getFaces() == &connectivity); + CHECK(triangle.getEdges() == nullptr); + CHECK(dataStructure.getData(k_GeometryPath.createChildPath(INodeGeometry1D::k_SharedEdgeListName)) == nullptr); + } + else + { + CHECK(geometry.getEdges() == &connectivity); + } +} +} // namespace + +TEST_CASE("SimplnxCore::LaplacianSmoothing: every node type uses its analytic lambda", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianNumerical]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(Catch::Generators::from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + const auto testCase = GENERATE(Catch::Generators::from_range(k_NodeTypeCases)); + const auto kind = GENERATE(GeometryKind::Triangle, GeometryKind::Edge); + DYNAMIC_SECTION(testCase.name << " on " << (kind == GeometryKind::Triangle ? "TriangleGeom" : "EdgeGeom")) + { + UnitTest::AlgorithmTestScope scope(scenario); + const SmoothingBudgetScope budget; + // Keep the original one-byte large-data threshold and force resident storage. + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + DataStructure dataStructure; + NodeTypes nodeTypes; + nodeTypes.fill(testCase.nodeType); + BuildGraph(dataStructure, kind, nodeTypes); + RequireResidentInputs(dataStructure); + ExecuteSmoothing(scope, dataStructure, 1, false); + CheckResult(dataStructure, kind, nodeTypes, testCase.expected); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("SimplnxCore::LaplacianSmoothing: iterations and Taubin match analytic coordinates", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianNumerical]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(Catch::Generators::from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + const auto testCase = GENERATE(Catch::Generators::from_range(k_IterationCases)); + const auto kind = GENERATE(GeometryKind::Triangle, GeometryKind::Edge); + DYNAMIC_SECTION(testCase.name << " on " << (kind == GeometryKind::Triangle ? "TriangleGeom" : "EdgeGeom")) + { + UnitTest::AlgorithmTestScope scope(scenario); + const SmoothingBudgetScope budget; + // Keep the original one-byte large-data threshold and force resident storage. + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + DataStructure dataStructure; + const NodeTypes nodeTypes{NodeType::Default, NodeType::Default, NodeType::Default, NodeType::Default}; + BuildGraph(dataStructure, kind, nodeTypes); + RequireResidentInputs(dataStructure); + ExecuteSmoothing(scope, dataStructure, testCase.iterations, testCase.useTaubin); + CheckResult(dataStructure, kind, nodeTypes, testCase.expected, testCase.iterations == 0); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("SimplnxCore::LaplacianSmoothing: mixed node classes use input-pass coordinates", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianNumerical]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(Catch::Generators::from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + const auto kind = GENERATE(GeometryKind::Triangle, GeometryKind::Edge); + DYNAMIC_SECTION((kind == GeometryKind::Triangle ? "TriangleGeom" : "EdgeGeom")) + { + UnitTest::AlgorithmTestScope scope(scenario); + const SmoothingBudgetScope budget; + // Keep the original one-byte large-data threshold and force resident storage. + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + DataStructure dataStructure; + const NodeTypes nodeTypes{NodeType::Unused, NodeType::Default, NodeType::TriplePoint, NodeType::Unused}; + const Coordinates expected{0.0F, 0.0F, 0.0F, 1.5F, 1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 3.0F, 1.0F, 1.0F / 3.0F, 0.0F, 0.0F, 2.0F}; + BuildGraph(dataStructure, kind, nodeTypes); + RequireResidentInputs(dataStructure); + ExecuteSmoothing(scope, dataStructure, 1, false); + CheckResult(dataStructure, kind, nodeTypes, expected); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} diff --git a/src/Plugins/SimplnxCore/test/LaplacianSmoothingTransferTest.cpp b/src/Plugins/SimplnxCore/test/LaplacianSmoothingTransferTest.cpp new file mode 100644 index 0000000000..dc1b5fb83a --- /dev/null +++ b/src/Plugins/SimplnxCore/test/LaplacianSmoothingTransferTest.cpp @@ -0,0 +1,526 @@ +#include "SimplnxCore/Filters/Algorithms/LaplacianSmoothing.hpp" +#include "SimplnxCore/Filters/LaplacianSmoothingFilter.hpp" + +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/Geometry/EdgeGeom.hpp" +#include "simplnx/DataStructure/Geometry/INodeGeometry2D.hpp" +#include "simplnx/DataStructure/IDataArray.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +/** @brief Stores the complete four-vertex coordinate oracle. */ +using Coordinates = std::array; +/** @brief Identifies the geometry changed by the public filter. */ +const DataPath k_GeometryPath{{"Mesh"}}; +/** @brief Identifies the coordinate payload supplied to writeback. */ +const DataPath k_VerticesPath{{"Mesh", "Vertices"}}; +/** @brief Identifies the four unchanged node classifications. */ +const DataPath k_NodeTypesPath{{"Mesh", "Vertex Data", "Node Types"}}; +/** @brief Identifies the six supplied edges retained by EdgeGeom. */ +const DataPath k_EdgesPath{{"Mesh", "Edges"}}; +/** @brief Supplies a tetrahedron with centroid (1/2, 1/2, 1/2). */ +constexpr Coordinates k_InputCoordinates{0.0F, 0.0F, 0.0F, 2.0F, 0.0F, 0.0F, 0.0F, 2.0F, 0.0F, 0.0F, 0.0F, 2.0F}; +/** @brief Supplies the one-pass result for lambda 1/4 on the complete four-vertex graph. */ +constexpr Coordinates k_PlainCoordinates{1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 6.0F, 1.5F, 1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 6.0F, 1.5F, 1.0F / 6.0F, 1.0F / 6.0F, 1.0F / 6.0F, 1.5F}; +/** @brief Supplies one Taubin pair with lambda 1/4 and mu factor -1. */ +constexpr Coordinates k_TaubinCoordinates{1.0F / 18.0F, 1.0F / 18.0F, 1.0F / 18.0F, 11.0F / 6.0F, 1.0F / 18.0F, 1.0F / 18.0F, + 1.0F / 18.0F, 11.0F / 6.0F, 1.0F / 18.0F, 1.0F / 18.0F, 1.0F / 18.0F, 11.0F / 6.0F}; +/** @brief Supplies every edge once; each vertex has degree three. */ +constexpr std::array k_Edges{0, 1, 0, 2, 0, 3, 1, 2, 1, 3, 2, 3}; +/** @brief Makes every vertex use the default lambda. */ +constexpr std::array k_NodeTypes{NodeType::Default, NodeType::Default, NodeType::Default, NodeType::Default}; +/** @brief Distinguishes the injected store failure from the algorithm's contextual error. */ +constexpr int32 k_InjectedError = -95241; +/** @brief Supplies the exact backend diagnostic that the filter must preserve. */ +constexpr const char* k_InjectedMessage = "Injected Laplacian smoothing bulk transfer failure."; + +/** + * @enum Transfer + * @brief Identifies reads and writes on the three resident fixture stores. + */ +enum class Transfer +{ + ReadNodes, ///< Reads vertex classifications. + ReadVertices, ///< Reads the original coordinates. + ReadEdges, ///< Reads the supplied connectivity. + WriteVertices, ///< Publishes completed coordinate passes. + WriteNodes, ///< Detects an unexpected classification write. + WriteEdges ///< Detects an unexpected connectivity write. +}; + +/** + * @struct TransferCall + * @brief Records the actual operation and flat value range requested by smoothing. + */ +struct TransferCall +{ + Transfer operation; + usize offset; + usize values; +}; + +/** @brief Specifies the complete expected bulk order; failures permit only its corresponding prefix. */ +constexpr std::array k_CompleteTransfers{{{Transfer::ReadNodes, 0, 4}, {Transfer::ReadVertices, 0, 12}, {Transfer::ReadEdges, 0, 12}, {Transfer::WriteVertices, 0, 12}}}; + +/** + * @struct FailureCase + * @brief Pairs a refused transfer with its contextual error and allowed call prefix. + */ +struct FailureCase +{ + const char* name; + Transfer operation; + int32 contextCode; + usize expectedCalls; +}; + +/** @brief Supplies independent stopping-order expectations for each bulk failure. */ +constexpr std::array k_FailureCases{ + {{"node read", Transfer::ReadNodes, -560, 1}, {"vertex read", Transfer::ReadVertices, -561, 2}, {"edge read", Transfer::ReadEdges, -562, 3}, {"vertex write", Transfer::WriteVertices, -563, 4}}}; + +/** + * @struct TransferTrace + * @brief Controls one synchronous fault and records calls only after fixture setup. + */ +struct TransferTrace +{ + bool armed = false; + std::optional failure; + std::vector calls; +}; + +/** + * @class RecordingStore + * @brief Uses real resident storage while recording and optionally refusing one bulk boundary. + * @tparam T Specifies the stored scalar type. + * @note Refusal happens before the underlying copy. It does not model rollback after partial backend I/O. + */ +template +class RecordingStore final : public DataStore +{ +public: + /** + * @brief Creates only the small resident fixture and borrows its longer-lived trace. + * @param tuples Supplies the fixture tuple count. + * @param components Supplies the values per tuple. + * @param trace Records calls after arming and outlives the store. + * @param readOperation Identifies reads from this input. + * @param writeOperation Identifies writes to this input. + */ + RecordingStore(usize tuples, usize components, TransferTrace& trace, Transfer readOperation, Transfer writeOperation) + : DataStore({tuples}, {components}, T{}) + , m_Trace(trace) + , m_ReadOperation(readOperation) + , m_WriteOperation(writeOperation) + { + } + + ~RecordingStore() override = default; + RecordingStore(const RecordingStore&) = default; + RecordingStore(RecordingStore&&) = default; + RecordingStore& operator=(const RecordingStore&) = delete; + RecordingStore& operator=(RecordingStore&&) = delete; + + /** + * @brief Records a bulk read and optionally returns the injected error before copying. + * @param startIndex Supplies the requested flat offset. + * @param buffer Receives values only when the read succeeds. + * @return The injected failure or the real resident copy result. + */ + [[nodiscard]] Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + if(m_Trace.armed) + { + m_Trace.calls.push_back({m_ReadOperation, startIndex, buffer.size()}); + if(m_Trace.failure == m_ReadOperation) + { + return MakeErrorResult(k_InjectedError, k_InjectedMessage); + } + } + return DataStore::copyIntoBuffer(startIndex, buffer); + } + + /** + * @brief Retains the attempted payload and optionally fails before resident values change. + * @param startIndex Supplies the requested flat offset. + * @param buffer Supplies the computed values offered to writeback. + * @return The injected failure or the real resident copy result. + */ + [[nodiscard]] Result<> copyFromBuffer(usize startIndex, nonstd::span buffer) override + { + if(m_Trace.armed) + { + m_Trace.calls.push_back({m_WriteOperation, startIndex, buffer.size()}); + attemptedWrite.assign(buffer.begin(), buffer.end()); + if(m_Trace.failure == m_WriteOperation) + { + return MakeErrorResult(k_InjectedError, k_InjectedMessage); + } + } + return DataStore::copyFromBuffer(startIndex, buffer); + } + + /** @brief Owns a copy of the last attempted write payload, including a refused write. */ + std::vector attemptedWrite; + +private: + TransferTrace& m_Trace; + const Transfer m_ReadOperation; + const Transfer m_WriteOperation; +}; + +/** + * @struct InputStores + * @brief Retains the actual stores whose attachments and values the tests verify. + */ +struct InputStores +{ + std::shared_ptr> vertices; + std::shared_ptr> nodes; + std::shared_ptr> edges; +}; + +/** + * @brief Builds a small EdgeGeom without incidental edge-generation reads. + * @param dataStructure Owns all fixture objects. + * @param trace Outlives the DataStructure and returned store references. + * @return References to the three resident fault/recording stores. + */ +InputStores BuildGraph(DataStructure& dataStructure, TransferTrace& trace) +{ + auto* geometryPtr = EdgeGeom::Create(dataStructure, k_GeometryPath.getTargetName()); + REQUIRE(geometryPtr != nullptr); + auto* vertexDataPtr = AttributeMatrix::Create(dataStructure, "Vertex Data", {4}, geometryPtr->getId()); + REQUIRE(vertexDataPtr != nullptr); + geometryPtr->setVertexAttributeMatrix(*vertexDataPtr); + InputStores stores{std::make_shared>(4, 3, trace, Transfer::ReadVertices, Transfer::WriteVertices), + std::make_shared>(4, 1, trace, Transfer::ReadNodes, Transfer::WriteNodes), + std::make_shared>(6, 2, trace, Transfer::ReadEdges, Transfer::WriteEdges)}; + auto* verticesPtr = Float32Array::Create(dataStructure, k_VerticesPath.getTargetName(), stores.vertices, geometryPtr->getId()); + auto* nodesPtr = Int8Array::Create(dataStructure, k_NodeTypesPath.getTargetName(), stores.nodes, vertexDataPtr->getId()); + auto* edgesPtr = IGeometry::MeshIndexArrayType::Create(dataStructure, k_EdgesPath.getTargetName(), stores.edges, geometryPtr->getId()); + REQUIRE(verticesPtr != nullptr); + REQUIRE(nodesPtr != nullptr); + REQUIRE(edgesPtr != nullptr); + geometryPtr->setVertices(*verticesPtr); + geometryPtr->setEdgeList(*edgesPtr); + const auto vertexResult = stores.vertices->copyFromBuffer(0, {k_InputCoordinates.data(), k_InputCoordinates.size()}); + const auto nodeResult = stores.nodes->copyFromBuffer(0, {k_NodeTypes.data(), k_NodeTypes.size()}); + const auto edgeResult = stores.edges->copyFromBuffer(0, {k_Edges.data(), k_Edges.size()}); + SIMPLNX_RESULT_REQUIRE_VALID(vertexResult); + SIMPLNX_RESULT_REQUIRE_VALID(nodeResult); + SIMPLNX_RESULT_REQUIRE_VALID(edgeResult); + REQUIRE(trace.calls.empty()); + return stores; +} + +/** + * @brief Supplies the public filter arguments with distinct weights for every node class. + * @param iterations Sets the requested outer pass count. + * @param useTaubin Enables the second pass with mu factor -1. + * @return Complete public filter arguments. + */ +Arguments MakeArguments(int32 iterations, bool useTaubin) +{ + Arguments args; + args.insertOrAssign(LaplacianSmoothingFilter::k_IterationSteps_Key, std::make_any(iterations)); + args.insertOrAssign(LaplacianSmoothingFilter::k_Lambda_Key, std::make_any(0.25F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_UseTaubinSmoothing_Key, std::make_any(useTaubin)); + args.insertOrAssign(LaplacianSmoothingFilter::k_MuFactor_Key, std::make_any(-1.0F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_TripleLineLambda_Key, std::make_any(0.5F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_QuadPointLambda_Key, std::make_any(0.75F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfacePointLambda_Key, std::make_any(0.125F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfaceTripleLineLambda_Key, std::make_any(0.375F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfaceQuadPointLambda_Key, std::make_any(0.625F)); + args.insertOrAssign(LaplacianSmoothingFilter::k_SurfaceMeshNodeTypeArrayPath_Key, std::make_any(k_NodeTypesPath)); + args.insertOrAssign(LaplacianSmoothingFilter::k_TriangleGeometryDataPath_Key, std::make_any(k_GeometryPath)); + return args; +} + +/** + * @brief Compares every actual call with the allowed prefix of the independent transfer sequence. + * @param trace Supplies calls made after arming. + * @param expectedCalls Supplies the prefix length, including the failed call when applicable. + */ +void CheckTransfers(const TransferTrace& trace, usize expectedCalls) +{ + REQUIRE(trace.calls.size() == expectedCalls); + REQUIRE(expectedCalls <= k_CompleteTransfers.size()); + for(usize callIdx = 0; callIdx < expectedCalls; ++callIdx) + { + CAPTURE(callIdx); + CHECK(trace.calls[callIdx].operation == k_CompleteTransfers[callIdx].operation); + CHECK(trace.calls[callIdx].offset == k_CompleteTransfers[callIdx].offset); + CHECK(trace.calls[callIdx].values == k_CompleteTransfers[callIdx].values); + } +} + +/** + * @brief Checks the complete payload offered to the coordinate write. + * @param actual Supplies the captured write buffer, including a refused write. + * @param expected Supplies the independently derived completed-pass coordinates. + */ +void CheckPayload(const std::vector& actual, const Coordinates& expected) +{ + REQUIRE(actual.size() == expected.size()); + for(usize valueIdx = 0; valueIdx < expected.size(); ++valueIdx) + { + CAPTURE(valueIdx, actual[valueIdx], expected[valueIdx]); + CHECK(std::abs(actual[valueIdx] - expected[valueIdx]) <= 1.0e-6F); + } +} + +/** + * @brief Verifies the stored coordinates, unchanged inputs, and original resident attachments. + * @param dataStructure Supplies the fixture before or after execution. + * @param stores Supplies the original three store objects. + * @param expected Supplies every expected coordinate. + * @param exact Requires exact coordinate equality when no successful write is permitted. + */ +void CheckGraph(const DataStructure& dataStructure, const InputStores& stores, const Coordinates& expected, bool exact) +{ + const EdgeGeom* geometryPtr = nullptr; + REQUIRE_NOTHROW(geometryPtr = &dataStructure.getDataRefAs(k_GeometryPath)); + const Float32Array* verticesPtr = nullptr; + REQUIRE_NOTHROW(verticesPtr = &dataStructure.getDataRefAs(k_VerticesPath)); + const Int8Array* nodesPtr = nullptr; + REQUIRE_NOTHROW(nodesPtr = &dataStructure.getDataRefAs(k_NodeTypesPath)); + const IGeometry::MeshIndexArrayType* edgesPtr = nullptr; + REQUIRE_NOTHROW(edgesPtr = &dataStructure.getDataRefAs(k_EdgesPath)); + CHECK(geometryPtr->getVertices() == verticesPtr); + CHECK(geometryPtr->getEdges() == edgesPtr); + CHECK(&verticesPtr->getDataStoreRef() == stores.vertices.get()); + CHECK(&nodesPtr->getDataStoreRef() == stores.nodes.get()); + CHECK(&edgesPtr->getDataStoreRef() == stores.edges.get()); + for(const auto& path : {k_VerticesPath, k_NodeTypesPath, k_EdgesPath}) + { + const IDataArray* arrayPtr = nullptr; + REQUIRE_NOTHROW(arrayPtr = &dataStructure.getDataRefAs(path)); + REQUIRE(arrayPtr->getIDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + } + REQUIRE(verticesPtr->getTupleShape() == ShapeType{4}); + REQUIRE(verticesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(nodesPtr->getTupleShape() == ShapeType{4}); + REQUIRE(nodesPtr->getComponentShape() == ShapeType{1}); + REQUIRE(edgesPtr->getTupleShape() == ShapeType{6}); + REQUIRE(edgesPtr->getComponentShape() == ShapeType{2}); + for(usize valueIdx = 0; valueIdx < expected.size(); ++valueIdx) + { + const float32 actual = verticesPtr->getValue(valueIdx); + CAPTURE(valueIdx, actual, expected[valueIdx]); + if(exact) + { + CHECK(actual == expected[valueIdx]); + } + else + { + CHECK(std::abs(actual - expected[valueIdx]) <= 1.0e-6F); + } + CHECK(edgesPtr->getValue(valueIdx) == k_Edges[valueIdx]); + } + for(usize vertexIdx = 0; vertexIdx < k_NodeTypes.size(); ++vertexIdx) + { + CAPTURE(vertexIdx); + CHECK(nodesPtr->getValue(vertexIdx) == k_NodeTypes[vertexIdx]); + } + CHECK(stores.nodes->attemptedWrite.empty()); + CHECK(stores.edges->attemptedWrite.empty()); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} +} // namespace + +TEST_CASE("SimplnxCore::LaplacianSmoothing: bulk failures stop at the failed transfer", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianTransfer]") +{ + UnitTest::LoadPlugins(); + const auto failure = GENERATE(Catch::Generators::from_range(k_FailureCases)); + CAPTURE(failure.name); + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + TransferTrace trace; + DataStructure dataStructure; + const auto stores = BuildGraph(dataStructure, trace); + CheckGraph(dataStructure, stores, k_InputCoordinates, true); + LaplacianSmoothingFilter filter; + const auto args = MakeArguments(1, false); + const auto preflight = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + trace.failure = failure.operation; + trace.armed = true; + const auto result = filter.execute(dataStructure, args).result; + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == k_InjectedError); + CHECK(result.errors()[0].message == k_InjectedMessage); + CHECK(result.warnings().empty()); + CHECK(result.errors()[1].code == failure.contextCode); + CHECK(result.errors()[1].message.find(failure.operation == Transfer::ReadNodes ? k_NodeTypesPath.toString() : k_GeometryPath.toString()) != std::string::npos); + CheckTransfers(trace, failure.expectedCalls); + if(failure.operation == Transfer::WriteVertices) + { + CheckPayload(stores.vertices->attemptedWrite, k_PlainCoordinates); + } + else + { + CHECK(stores.vertices->attemptedWrite.empty()); + } + CheckGraph(dataStructure, stores, k_InputCoordinates, true); +} + +TEST_CASE("SimplnxCore::LaplacianSmoothing: cancellation publishes completed passes and retains write failures", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianCancellation]") +{ + UnitTest::LoadPlugins(); + const std::string schedule = GENERATE("after plain pass", "before Taubin second pass", "after Taubin pair"); + const bool failWrite = GENERATE(false, true); + CAPTURE(schedule, failWrite); + const bool useTaubin = schedule != "after plain pass"; + const auto& expected = schedule == "after Taubin pair" ? k_TaubinCoordinates : k_PlainCoordinates; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + TransferTrace trace; + DataStructure dataStructure; + const auto stores = BuildGraph(dataStructure, trace); + CheckGraph(dataStructure, stores, k_InputCoordinates, true); + LaplacianSmoothingFilter filter; + const auto args = MakeArguments(2, useTaubin); + const auto preflight = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + std::atomic_bool shouldCancel = false; + std::vector iterations; + const IFilter::MessageHandler handler{[&](const IFilter::Message& message) { + if(message.type == IFilter::Message::Type::Progress && message.message.starts_with("Smoothing Iterations: ")) + { + iterations.push_back(message.message); + const bool atCheckpoint = schedule == "before Taubin second pass" ? message.message.find("Starting Taubin pass") != std::string::npos : message.message.starts_with("Smoothing Iterations: 1/2"); + if(atCheckpoint) + { + shouldCancel.store(true); + } + } + }}; + if(failWrite) + { + trace.failure = Transfer::WriteVertices; + } + trace.armed = true; + const auto result = filter.execute(dataStructure, args, nullptr, handler, shouldCancel).result; + REQUIRE(shouldCancel.load()); + REQUIRE(result.invalid()); + if(failWrite) + { + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == k_InjectedError); + CHECK(result.errors()[0].message == k_InjectedMessage); + CHECK(result.errors()[1].code == -563); + CHECK(result.errors()[1].message.find(k_GeometryPath.toString()) != std::string::npos); + } + else + { + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -1); + } + CHECK(result.warnings().empty()); + REQUIRE(iterations.size() == (schedule == "after Taubin pair" ? 3 : 2)); + CHECK(iterations.front() == "Smoothing Iterations: 0/2"); + if(schedule == "before Taubin second pass") + { + CHECK(iterations.back().starts_with("Smoothing Iterations: 0/2")); + CHECK(iterations.back().find("Starting Taubin pass") != std::string::npos); + } + else + { + CHECK(iterations.back().starts_with("Smoothing Iterations: 1/2")); + } + CheckTransfers(trace, k_CompleteTransfers.size()); + CheckPayload(stores.vertices->attemptedWrite, expected); + CheckGraph(dataStructure, stores, failWrite ? k_InputCoordinates : expected, failWrite); +} + +TEST_CASE("SimplnxCore::LaplacianSmoothing: zero iterations and pre-cancellation do not write coordinates", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianTransfer]") +{ + UnitTest::LoadPlugins(); + const std::string scenario = GENERATE("zero plain iterations", "zero Taubin iterations", "pre-cancelled"); + CAPTURE(scenario); + const bool preCancelled = scenario == "pre-cancelled"; + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + TransferTrace trace; + DataStructure dataStructure; + const auto stores = BuildGraph(dataStructure, trace); + CheckGraph(dataStructure, stores, k_InputCoordinates, true); + LaplacianSmoothingFilter filter; + const auto args = MakeArguments(preCancelled ? 2 : 0, scenario == "zero Taubin iterations"); + const auto preflight = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + const std::atomic_bool shouldCancel = preCancelled; + trace.armed = true; + const auto result = filter.execute(dataStructure, args, nullptr, {}, shouldCancel).result; + if(preCancelled) + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -1); + } + else + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + CHECK(result.warnings().empty()); + if(preCancelled) + { + CheckTransfers(trace, 0); + } + else + { + // A zero-iteration implementation may skip the reads; it must never publish a coordinate write. + for(const auto& call : trace.calls) + { + CHECK(call.operation != Transfer::WriteVertices); + CHECK(call.operation != Transfer::WriteNodes); + CHECK(call.operation != Transfer::WriteEdges); + } + } + CHECK(stores.vertices->attemptedWrite.empty()); + CheckGraph(dataStructure, stores, k_InputCoordinates, true); +} + +TEST_CASE("SimplnxCore::LaplacianSmoothing: missing vertices fail the direct algorithm guard", "[SimplnxCore][LaplacianSmoothingFilter][LaplacianTransfer]") +{ + UnitTest::LoadPlugins(); + const UnitTest::PreferencesSentinel resident(DataStorageMode::ForceInCore, 1); + DataStructure dataStructure; + auto* geometryPtr = EdgeGeom::Create(dataStructure, k_GeometryPath.getTargetName()); + REQUIRE(geometryPtr != nullptr); + REQUIRE(geometryPtr->getVertices() == nullptr); + LaplacianSmoothingInputValues values{}; + values.pTriangleGeometryDataPath = k_GeometryPath; + values.pSurfaceMeshNodeTypeArrayPath = k_NodeTypesPath; + const std::atomic_bool shouldCancel = false; + const IFilter::MessageHandler handler{}; + Result<> result; + // Public parameter validation rejects the absent node array before this distinct algorithm guard can run. + REQUIRE_NOTHROW(result = LaplacianSmoothing(dataStructure, &values, shouldCancel, handler)()); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -559); + CHECK(result.errors()[0].message.find(k_GeometryPath.toString()) != std::string::npos); + CHECK(result.warnings().empty()); + CHECK(geometryPtr->getVertices() == nullptr); + CHECK(geometryPtr->getEdges() == nullptr); + CHECK(dataStructure.getData(k_VerticesPath) == nullptr); + CHECK(dataStructure.getData(k_NodeTypesPath) == nullptr); + CHECK(dataStructure.getData(k_EdgesPath) == nullptr); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} diff --git a/src/Plugins/SimplnxCore/test/RepairTriangleWindingTransferTest.cpp b/src/Plugins/SimplnxCore/test/RepairTriangleWindingTransferTest.cpp new file mode 100644 index 0000000000..bb24c37f43 --- /dev/null +++ b/src/Plugins/SimplnxCore/test/RepairTriangleWindingTransferTest.cpp @@ -0,0 +1,299 @@ +#include "simplnx/Common/Result.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/DynamicListArray.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" +#include "simplnx/Filter/IFilter.hpp" +#include "simplnx/Utilities/Meshing/TriangleUtilities.hpp" + +#include + +#include +#include +#include + +using namespace nx::core; + +namespace +{ +using ConnectivityType = std::array; +using NeighborsType = DynamicListArray; + +// Each pair starts with the same directed shared edge. Reversing the second face resolves that conflict. +constexpr ConnectivityType k_InputTriangles{0, 1, 2, 0, 1, 3, 4, 5, 6, 4, 5, 7}; +constexpr ConnectivityType k_RepairedTriangles{0, 1, 2, 3, 1, 0, 4, 5, 6, 7, 5, 4}; +constexpr ConnectivityType k_PartialTriangles{0, 1, 2, 3, 1, 0, 4, 5, 6, 4, 5, 7}; +constexpr std::array k_NeighborIds{1, 0, 3, 2}; + +enum class Transfer +{ + ReadTriangles, + ReadIds, + WriteTriangles, + WriteIds +}; + +struct TransferCall +{ + Transfer operation; + usize offset; + usize values; +}; + +struct InjectedFailure +{ + Transfer operation; + int32 code; + const char* message; +}; + +constexpr InjectedFailure k_TriangleReadFailure{Transfer::ReadTriangles, -95251, "Injected triangle winding connectivity read refusal."}; +constexpr InjectedFailure k_IdReadFailure{Transfer::ReadIds, -95252, "Injected triangle winding ID read refusal."}; +constexpr InjectedFailure k_TriangleWriteFailure{Transfer::WriteTriangles, -95253, "Injected triangle winding connectivity write refusal."}; + +struct TransferTrace +{ + bool armed = false; + const InjectedFailure* failure = nullptr; + std::vector calls; + std::vector rejectedCalls; +}; + +/** + * @class RecordingStore + * @brief Records transfers on real resident storage and refuses selected calls before copying. + * @tparam T Specifies the stored scalar type. + * @note Refusal does not model rollback after partial backend I/O. + */ +template +class RecordingStore final : public DataStore +{ +public: + /** + * @brief Creates a four-triangle input store and borrows its longer-lived trace. + * @param components Specifies the values per triangle. + * @param trace Records calls after setup and outlives the store. + * @param readOperation Identifies this store's reads. + * @param writeOperation Identifies this store's writes. + */ + RecordingStore(usize components, TransferTrace& trace, Transfer readOperation, Transfer writeOperation) + : DataStore({4}, {components}, T{}) + , m_Trace(trace) + , m_ReadOperation(readOperation) + , m_WriteOperation(writeOperation) + { + } + + ~RecordingStore() override = default; + RecordingStore(const RecordingStore&) = default; + RecordingStore(RecordingStore&&) = default; + RecordingStore& operator=(const RecordingStore&) = delete; + RecordingStore& operator=(RecordingStore&&) = delete; + + /** + * @brief Records a read and refuses the selected call before changing the destination. + * @param startIndex Specifies the first flat value index. + * @param buffer Receives values only when the read succeeds. + * @return The injected failure or the resident copy result. + */ + [[nodiscard]] Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + if(m_Trace.armed) + { + m_Trace.calls.push_back({m_ReadOperation, startIndex, buffer.size()}); + if(m_Trace.failure != nullptr && m_Trace.failure->operation == m_ReadOperation) + { + m_Trace.rejectedCalls.push_back(m_ReadOperation); + return MakeErrorResult(m_Trace.failure->code, m_Trace.failure->message); + } + } + return DataStore::copyIntoBuffer(startIndex, buffer); + } + + /** + * @brief Saves the attempted payload and refuses the selected write before changing resident values. + * @param startIndex Specifies the first flat value index. + * @param buffer Supplies the computed values offered for publication. + * @return The injected failure or the resident copy result. + */ + [[nodiscard]] Result<> copyFromBuffer(usize startIndex, nonstd::span buffer) override + { + if(m_Trace.armed) + { + m_Trace.calls.push_back({m_WriteOperation, startIndex, buffer.size()}); + attemptedWrite.assign(buffer.begin(), buffer.end()); + if(m_Trace.failure != nullptr && m_Trace.failure->operation == m_WriteOperation) + { + m_Trace.rejectedCalls.push_back(m_WriteOperation); + return MakeErrorResult(m_Trace.failure->code, m_Trace.failure->message); + } + } + return DataStore::copyFromBuffer(startIndex, buffer); + } + + // The payload survives a refused write so the test can inspect the computed connectivity. + std::vector attemptedWrite; + +private: + TransferTrace& m_Trace; + const Transfer m_ReadOperation; + const Transfer m_WriteOperation; +}; + +struct WindingCase +{ + const char* name; + const InjectedFailure* failure; + bool preCancelled; + bool cancelAfterFeature; + usize expectedCalls; + const ConnectivityType* expectedStoredTriangles; + const ConnectivityType* expectedWrittenTriangles; +}; + +constexpr std::array k_TransferCases{{ + {"successful control", nullptr, false, false, 3, &k_RepairedTriangles, &k_RepairedTriangles}, + {"connectivity read refusal", &k_TriangleReadFailure, false, false, 1, &k_InputTriangles, nullptr}, + {"ID read refusal", &k_IdReadFailure, false, false, 2, &k_InputTriangles, nullptr}, + {"connectivity write refusal", &k_TriangleWriteFailure, false, false, 3, &k_InputTriangles, &k_RepairedTriangles}, +}}; + +constexpr std::array k_CancellationCases{{ + {"pre-cancelled control", nullptr, true, false, 3, &k_InputTriangles, &k_InputTriangles}, + {"cancellation after feature 1", nullptr, false, true, 3, &k_PartialTriangles, &k_PartialTriangles}, + {"write refusal after feature 1 cancellation", &k_TriangleWriteFailure, false, true, 3, &k_InputTriangles, &k_PartialTriangles}, +}}; + +/** + * @brief Runs one direct utility call with valid topology and independently checks transfers and values. + * @param useFaceLabels Selects two-component face labels instead of one-component regions. + * @param windingCase Supplies independent transfer, result, and connectivity expectations. + */ +void CheckWindingCase(bool useFaceLabels, const WindingCase& windingCase) +{ + // The trace and DataStructure outlive the stores and neighbor reference used by the utility. + TransferTrace trace; + DataStructure dataStructure; + RecordingStore triangles(3, trace, Transfer::ReadTriangles, Transfer::WriteTriangles); + RecordingStore ids(useFaceLabels ? 2 : 1, trace, Transfer::ReadIds, Transfer::WriteIds); + const std::vector inputIds = useFaceLabels ? std::vector{0, 1, 0, 1, 0, 2, 0, 2} : std::vector{1, 1, 2, 2}; + auto* neighborsPtr = NeighborsType::Create(dataStructure, "Neighbors", {}); + REQUIRE(neighborsPtr != nullptr); + neighborsPtr->allocateLists(std::array{1, 1, 1, 1}); + for(usize triangleIdx = 0; triangleIdx < k_NeighborIds.size(); ++triangleIdx) + { + neighborsPtr->insertCellReference(triangleIdx, 0, k_NeighborIds[triangleIdx]); + } + + const auto triangleSeedResult = triangles.copyFromBuffer(0, {k_InputTriangles.data(), k_InputTriangles.size()}); + const auto idSeedResult = ids.copyFromBuffer(0, {inputIds.data(), inputIds.size()}); + REQUIRE(triangleSeedResult.valid()); + REQUIRE(idSeedResult.valid()); + REQUIRE(triangles.getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(ids.getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(trace.calls.empty()); + REQUIRE(trace.rejectedCalls.empty()); + REQUIRE(triangles.attemptedWrite.empty()); + REQUIRE(ids.attemptedWrite.empty()); + + std::atomic_bool shouldCancel = windingCase.preCancelled; + usize cancellationWitnesses = 0; + const IFilter::MessageHandler handler{IFilter::MessageHandler::Callback{[&](const IFilter::Message& message) { + // reset() opens the first progress gate; this synchronous callback stops traversal before feature 2. + if(windingCase.cancelAfterFeature && message.type == IFilter::Message::Type::Progress && message.progress == 50 && message.message.rfind("Repairing Feature Windings", 0) == 0) + { + ++cancellationWitnesses; + shouldCancel.store(true); + } + }}}; + + trace.failure = windingCase.failure; + trace.armed = true; + const auto result = MeshingUtilities::RepairTriangleWinding(triangles, *neighborsPtr, ids, shouldCancel, handler); + trace.armed = false; + + REQUIRE(result.warnings().empty()); + if(windingCase.failure == nullptr) + { + REQUIRE(result.valid()); + REQUIRE(trace.rejectedCalls.empty()); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.errors()[0].code == windingCase.failure->code); + REQUIRE(result.errors()[0].message == windingCase.failure->message); + REQUIRE(trace.rejectedCalls.size() == 1); + REQUIRE(trace.rejectedCalls[0] == windingCase.failure->operation); + } + + const std::array expectedTransfers{{{Transfer::ReadTriangles, 0, 12}, {Transfer::ReadIds, 0, inputIds.size()}, {Transfer::WriteTriangles, 0, 12}}}; + REQUIRE(trace.calls.size() == windingCase.expectedCalls); + REQUIRE(windingCase.expectedCalls <= expectedTransfers.size()); + for(usize callIdx = 0; callIdx < windingCase.expectedCalls; ++callIdx) + { + CAPTURE(callIdx); + REQUIRE(trace.calls[callIdx].operation == expectedTransfers[callIdx].operation); + REQUIRE(trace.calls[callIdx].offset == expectedTransfers[callIdx].offset); + REQUIRE(trace.calls[callIdx].values == expectedTransfers[callIdx].values); + } + REQUIRE(cancellationWitnesses == (windingCase.cancelAfterFeature ? 1 : 0)); + REQUIRE(shouldCancel.load() == (windingCase.preCancelled || windingCase.cancelAfterFeature)); + + REQUIRE(triangles.getSize() == windingCase.expectedStoredTriangles->size()); + for(usize valueIdx = 0; valueIdx < triangles.getSize(); ++valueIdx) + { + CAPTURE(valueIdx); + REQUIRE(triangles[valueIdx] == (*windingCase.expectedStoredTriangles)[valueIdx]); + } + if(windingCase.expectedWrittenTriangles == nullptr) + { + REQUIRE(triangles.attemptedWrite.empty()); + } + else + { + REQUIRE(triangles.attemptedWrite.size() == windingCase.expectedWrittenTriangles->size()); + for(usize valueIdx = 0; valueIdx < triangles.attemptedWrite.size(); ++valueIdx) + { + CAPTURE(valueIdx); + REQUIRE(triangles.attemptedWrite[valueIdx] == (*windingCase.expectedWrittenTriangles)[valueIdx]); + } + } + REQUIRE(ids.attemptedWrite.empty()); + REQUIRE(ids.getSize() == inputIds.size()); + for(usize valueIdx = 0; valueIdx < inputIds.size(); ++valueIdx) + { + CAPTURE(valueIdx); + REQUIRE(ids[valueIdx] == inputIds[valueIdx]); + } + for(usize triangleIdx = 0; triangleIdx < k_NeighborIds.size(); ++triangleIdx) + { + CAPTURE(triangleIdx); + REQUIRE(neighborsPtr->getNumberOfElements(triangleIdx) == 1); + const auto* neighborListPtr = neighborsPtr->getElementListPointer(triangleIdx); + REQUIRE(neighborListPtr != nullptr); + REQUIRE(neighborListPtr[0] == k_NeighborIds[triangleIdx]); + } +} +} // namespace + +TEST_CASE("SimplnxCore::RepairTriangleWinding: buffered transfer results", "[SimplnxCore][RepairTriangleWinding][ooc-review-winding-transfer]") +{ + const bool useFaceLabels = GENERATE(false, true); + const auto windingCase = GENERATE(from_range(k_TransferCases)); + DYNAMIC_SECTION((useFaceLabels ? "face labels" : "regions") << ": " << windingCase.name) + { + CheckWindingCase(useFaceLabels, windingCase); + } +} + +TEST_CASE("SimplnxCore::RepairTriangleWinding: buffered cancellation publication", "[SimplnxCore][RepairTriangleWinding][ooc-review-winding-transfer]") +{ + const bool useFaceLabels = GENERATE(false, true); + const auto windingCase = GENERATE(from_range(k_CancellationCases)); + DYNAMIC_SECTION((useFaceLabels ? "face labels" : "regions") << ": " << windingCase.name) + { + CheckWindingCase(useFaceLabels, windingCase); + } +} diff --git a/src/Plugins/SimplnxCore/test/SharedFeatureFaceTest.cpp b/src/Plugins/SimplnxCore/test/SharedFeatureFaceTest.cpp index 30c1499d65..498fd4ddcc 100644 --- a/src/Plugins/SimplnxCore/test/SharedFeatureFaceTest.cpp +++ b/src/Plugins/SimplnxCore/test/SharedFeatureFaceTest.cpp @@ -1,13 +1,25 @@ #include "SimplnxCore/Filters/SharedFeatureFaceFilter.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataPath.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" +#include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/Filter/Arguments.hpp" #include "simplnx/Parameters/ArrayCreationParameter.hpp" #include "simplnx/Parameters/DataObjectNameParameter.hpp" #include "simplnx/Parameters/GeometrySelectionParameter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" #include +#include +#include +#include + using namespace nx::core; using namespace nx::core::UnitTest; @@ -35,12 +47,17 @@ using namespace SharedFeatureFaceFilterTest; TEST_CASE("SimplnxCore::SharedFeatureFaceFilter", "[SimplnxCore][SharedFeatureFaceFilter]") { UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_TestFilesDir, "12_IN625_GBCD.tar.gz", "12_IN625_GBCD"); // Read Exemplar DREAM3D File Filter auto exemplarFilePath = fs::path(fmt::format("{}/12_IN625_GBCD/12_IN625_GBCD.dream3d", unit_test::k_TestFilesDir)); DataStructure dataStructure = LoadDataStructure(exemplarFilePath); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_FaceLabelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(k_FaceLabelsPath)); { // Instantiate the filter and an Arguments Object @@ -61,7 +78,7 @@ TEST_CASE("SimplnxCore::SharedFeatureFaceFilter", "[SimplnxCore][SharedFeatureFa SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); // Execute the filter and check the result - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); } @@ -73,6 +90,7 @@ TEST_CASE("SimplnxCore::SharedFeatureFaceFilter", "[SimplnxCore][SharedFeatureFa const auto& kExemplarsArray = dataStructure.getDataRefAs(kExemplarArrayPath); const auto& kNxArray = dataStructure.getDataRefAs(kNxArrayPath); + scope.requireExpectedStore(kNxArray); UnitTest::CompareDataArrays(kExemplarsArray, kNxArray); } @@ -83,6 +101,7 @@ TEST_CASE("SimplnxCore::SharedFeatureFaceFilter", "[SimplnxCore][SharedFeatureFa const auto& kExemplarsArray = dataStructure.getDataRefAs(kExemplarArrayPath); const auto& kNxArray = dataStructure.getDataRefAs(kNxArrayPath); + scope.requireExpectedStore(kNxArray); UnitTest::CompareDataArrays(kExemplarsArray, kNxArray); } @@ -93,6 +112,7 @@ TEST_CASE("SimplnxCore::SharedFeatureFaceFilter", "[SimplnxCore][SharedFeatureFa const auto& kExemplarsArray = dataStructure.getDataRefAs(kExemplarArrayPath); const auto& kNxArray = dataStructure.getDataRefAs(kNxArrayPath); + scope.requireExpectedStore(kNxArray); UnitTest::CompareDataArrays(kExemplarsArray, kNxArray); } @@ -103,3 +123,71 @@ TEST_CASE("SimplnxCore::SharedFeatureFaceFilter", "[SimplnxCore][SharedFeatureFa UnitTest::CheckArraysInheritTupleDims(dataStructure); } + +TEST_CASE("SimplnxCore::SharedFeatureFaceFilter: hand-counted labels and reserved row", "[SimplnxCore][SharedFeatureFaceFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool realZeroPair = GENERATE(false, true); + CAPTURE(scenario, realZeroPair); + UnitTest::AlgorithmTestScope scope(scenario); + DataStructure dataStructure; + const DataPath geometryPath({"Mesh"}); + const DataPath faceDataPath = geometryPath.createChildPath("FaceData"); + const DataPath labelsPath = faceDataPath.createChildPath("Labels"); + auto* geometryPtr = TriangleGeom::Create(dataStructure, "Mesh"); + REQUIRE(geometryPtr != nullptr); + auto* faceDataPtr = AttributeMatrix::Create(dataStructure, "FaceData", {6}, geometryPtr->getId()); + REQUIRE(faceDataPtr != nullptr); + geometryPtr->setFaceAttributeMatrix(*faceDataPtr); + auto connectivityStore = DataStoreUtilities::CreateDataStore(dataStructure, geometryPath.createChildPath("Faces"), {6}, {3}); + auto* connectivityPtr = DataArray::Create(dataStructure, "Faces", connectivityStore, geometryPtr->getId()); + REQUIRE(connectivityPtr != nullptr); + geometryPtr->setFaceList(*connectivityPtr); + auto inputStore = DataStoreUtilities::CreateDataStore(dataStructure, labelsPath, {6}, {2}); + auto* inputPtr = Int32Array::Create(dataStructure, "Labels", inputStore, faceDataPtr->getId()); + REQUIRE(inputPtr != nullptr); + const std::array inputLabels{2, 1, 1, 2, 3, -1, 9, 4, 4, 9, 0, realZeroPair ? 0 : 5}; + REQUIRE(inputStore->copyFromBuffer(0, {inputLabels.data(), inputLabels.size()}).valid()); + scope.requireExpectedStore(*inputPtr); + SharedFeatureFaceFilter filter; + Arguments args; + args.insertOrAssign(SharedFeatureFaceFilter::k_TriGeometryDataPath_Key, std::make_any(geometryPath)); + args.insertOrAssign(SharedFeatureFaceFilter::k_FaceLabelsArrayPath_Key, std::make_any(labelsPath)); + args.insertOrAssign(SharedFeatureFaceFilter::k_FeatureFaceIdsArrayName_Key, std::make_any("Ids")); + args.insertOrAssign(SharedFeatureFaceFilter::k_GrainBoundaryAttributeMatrixName_Key, std::make_any("FeatureData")); + args.insertOrAssign(SharedFeatureFaceFilter::k_FeatureNumTrianglesArrayName_Key, std::make_any("Counts")); + args.insertOrAssign(SharedFeatureFaceFilter::k_FeatureFaceLabelsArrayName_Key, std::make_any("Pairs")); + args.insertOrAssign(SharedFeatureFaceFilter::k_RandomizeFeatures_Key, std::make_any(false)); + const auto result = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + + const DataPath idsPath = faceDataPath.createChildPath("Ids"); + const DataPath pairsPath = geometryPath.createChildPath("FeatureData").createChildPath("Pairs"); + const DataPath countsPath = geometryPath.createChildPath("FeatureData").createChildPath("Counts"); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(idsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(pairsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(countsPath)); + const auto& ids = dataStructure.getDataRefAs(idsPath); + const auto& pairs = dataStructure.getDataRefAs(pairsPath); + const auto& counts = dataStructure.getDataRefAs(countsPath); + scope.requireExpectedStore(ids); + scope.requireExpectedStore(pairs); + scope.requireExpectedStore(counts); + REQUIRE(ids.getSize() == 6); + REQUIRE(pairs.getSize() == 10); + REQUIRE(counts.getSize() == 5); + std::array actualIds{}; + std::array actualPairs{}; + std::array actualCounts{}; + REQUIRE(ids.getDataStoreRef().copyIntoBuffer(0, {actualIds.data(), actualIds.size()}).valid()); + REQUIRE(pairs.getDataStoreRef().copyIntoBuffer(0, {actualPairs.data(), actualPairs.size()}).valid()); + REQUIRE(counts.getDataStoreRef().copyIntoBuffer(0, {actualCounts.data(), actualCounts.size()}).valid()); + const std::array expectedIds{1, 1, 2, 3, 3, 4}; + const std::array expectedPairs{0, 0, 1, 2, -1, 3, 4, 9, 0, realZeroPair ? 0 : 5}; + const std::array expectedCounts{realZeroPair ? 1 : 0, 2, 1, 2, 1}; + CHECK(actualIds == expectedIds); + CHECK(actualPairs == expectedPairs); + CHECK(actualCounts == expectedCounts); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} diff --git a/src/Plugins/SimplnxCore/test/SimplnxCore_test_main.cpp b/src/Plugins/SimplnxCore/test/SimplnxCore_test_main.cpp index 48425c5326..6b322c3f17 100644 --- a/src/Plugins/SimplnxCore/test/SimplnxCore_test_main.cpp +++ b/src/Plugins/SimplnxCore/test/SimplnxCore_test_main.cpp @@ -1,4 +1,11 @@ // Catch2 recommends placing these lines by themselves in a translation unit // which will help reduce unnecessary recompilations of the expensive Catch main -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_RUNNER #include + +#include "simplnx/UnitTest/OwnedCatchRunner.hpp" + +int main(int argc, char* argv[]) +{ + return nx::core::UnitTest::RunOwnedCatchSession(argc, argv); +} diff --git a/src/Plugins/SimplnxCore/test/TriangleDihedralAngleFilterTest.cpp b/src/Plugins/SimplnxCore/test/TriangleDihedralAngleFilterTest.cpp index 12a3f606b6..a3562feac8 100644 --- a/src/Plugins/SimplnxCore/test/TriangleDihedralAngleFilterTest.cpp +++ b/src/Plugins/SimplnxCore/test/TriangleDihedralAngleFilterTest.cpp @@ -1,15 +1,31 @@ #include "SimplnxCore/Filters/TriangleDihedralAngleFilter.hpp" +#include "SimplnxCore/Filters/Algorithms/TriangleDihedralAngle.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" #include "simplnx/Core/Application.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" #include "simplnx/Parameters/ArrayCreationParameter.hpp" #include "simplnx/Parameters/FileSystemPathParameter.hpp" #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" #include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -50,6 +66,9 @@ static const IGeometry::SharedFaceList* createFaceList(IGeometry& geom, const Da TEST_CASE("SimplnxCore::TriangleDihedralAngleFilter[valid results]", "[SimplnxCore][TriangleDihedralAngleFilter]") { UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); std::string triangleGeometryName = "[Triangle Geometry]"; std::string triangleFaceDataGroupName = "Face Data"; @@ -89,6 +108,12 @@ TEST_CASE("SimplnxCore::TriangleDihedralAngleFilter[valid results]", "[SimplnxCo DataPath geometryPath = dataStructure.getDataPathsForId(acuteTriangle.getId())[0]; DataPath triangleDihedralAnglesDataPath = geometryPath.createChildPath(INodeGeometry2D::k_FaceAttributeMatrixName).createChildPath(triangleDihedralAnglesName); + const auto* verticesArray = acuteTriangle.getVertices(); + const auto* facesArray = acuteTriangle.getFaces(); + REQUIRE(verticesArray != nullptr); + REQUIRE(facesArray != nullptr); + scope.requireExpectedStore(*verticesArray); + scope.requireExpectedStore(*facesArray); // Create default Parameters for the filter. args.insertOrAssign(TriangleDihedralAngleFilter::k_TGeometryDataPath_Key, std::make_any(geometryPath)); @@ -99,9 +124,12 @@ TEST_CASE("SimplnxCore::TriangleDihedralAngleFilter[valid results]", "[SimplnxCo SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions) // Execute the filter and check the result - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result) + REQUIRE_NOTHROW(dataStructure.getDataRefAs(triangleDihedralAnglesDataPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(triangleDihedralAnglesDataPath)); + std::vector officialDihedralAngleMinimums = {53.1301002502441, 18.4349555969238, 0.0, 0.0}; Float64Array& calculatedDihedralAngleMinimums = dataStructure.getDataRefAs(triangleDihedralAnglesDataPath); @@ -159,3 +187,580 @@ TEST_CASE("SimplnxCore::TriangleDihedralAngleFilter: SIMPL Backwards Compatibili } } } + +namespace +{ +constexpr usize k_DihedralBlockFaces = 262144; +constexpr usize k_DihedralBufferFaces = 4096; +constexpr float64 k_DihedralSentinel = -37.0; +constexpr int32 k_DihedralTransferError = -98761; +const DataPath k_DihedralGeometryPath({"DihedralMesh"}); +const DataPath k_DihedralVerticesPath = k_DihedralGeometryPath.createChildPath("Vertices"); +const DataPath k_DihedralFacesPath = k_DihedralGeometryPath.createChildPath("Faces"); +const DataPath k_DihedralOutputPath = k_DihedralGeometryPath.createChildPath("FaceData").createChildPath("Angles"); +constexpr std::array k_DihedralVertices = {0, 0, 0, 1, 0, 0, 0.5f, 1, 0, 2, 0, 0, 3, 0, 0, 4, 1, 0, 6, 0, 0, 7, 0, 0, 8, 0, 0, 9, 0, 0, 10, 0, 0}; +constexpr std::array, 4> k_DihedralFaces = {{{0, 1, 2}, {3, 4, 5}, {6, 7, 7}, {8, 9, 10}}}; +// The finite angles in degrees are 2*atan(1/2), atan(1/3), and zero for the collinear face. +// The repeated-vertex face must produce NaN, which has a separate comparison below. +constexpr std::array k_ExpectedAngles = {53.13010235415598, 18.43494882292201, 0.0, 0.0}; + +usize DihedralFixtureRow(usize faceIndex) +{ + return faceIndex == k_DihedralBlockFaces ? 1 : faceIndex % 4; +} + +/** + * @brief Compares one angle without accepting NaN as a finite or unpublished value. + * @param actual Stored angle, in degrees. + * @param row Identifies the independent triangle fixture. + * @param completed Selects a computed angle or the unchanged output sentinel. + * @return True if the scalar has the required value or NaN classification. + */ +bool MatchesDihedralAngle(float64 actual, usize row, bool completed) +{ + if(!completed) + { + return actual == k_DihedralSentinel; + } + if(row == 2) + { + return std::isnan(actual); + } + if(row == 3) + { + return actual == 0.0; + } + return std::isfinite(actual) && std::abs(actual - k_ExpectedAngles[row]) <= 1.0e-10; +} + +enum class DihedralStoreRole : usize +{ + Vertices, + Faces, + Angles +}; + +struct DihedralTransfer +{ + DihedralStoreRole role; + bool write; + usize start; + usize count; +}; + +struct DihedralTransferState +{ + bool armed = false; + DihedralStoreRole failingRole = DihedralStoreRole::Angles; + bool failingWrite = false; + usize failingCall = 0; + bool cancelAfterFirstWrite = false; + std::atomic_bool shouldCancel{false}; + std::array reads{}; + std::array writes{}; + std::vector calls; +}; + +const char* DihedralFailureMessage(DihedralStoreRole role) +{ + switch(role) + { + case DihedralStoreRole::Vertices: + return "Injected triangle dihedral angle vertex read failure."; + case DihedralStoreRole::Faces: + return "Injected triangle dihedral angle connectivity read failure."; + case DihedralStoreRole::Angles: + return "Injected triangle dihedral angle output write failure."; + } + return "Unknown triangle dihedral angle transfer failure."; +} + +/** + * @class DihedralReportingStore + * @brief Records sequential bulk transfers and rejects selected calls before touching resident values. + * @tparam T Stored scalar type. + * @note The borrowed state outlives the store. This fixture does not model partial backend writes. + */ +template +class DihedralReportingStore final : public DataStore +{ +public: + /** + * @brief Creates real resident storage with a disarmed, longer-lived trace. + * @param tuples Number of tuples; output tuples have one component. + * @param state Records target transfers only. + * @param role Identifies the array in each transfer. + */ + DihedralReportingStore(usize tuples, DihedralTransferState& state, DihedralStoreRole role) + : DataStore(ShapeType{tuples}, ShapeType{role == DihedralStoreRole::Angles ? usize{1} : usize{3}}, T{}) + , m_State(state) + , m_Role(role) + { + } + + ~DihedralReportingStore() override = default; + DihedralReportingStore(const DihedralReportingStore&) = default; + DihedralReportingStore(DihedralReportingStore&&) = default; + DihedralReportingStore& operator=(const DihedralReportingStore&) = delete; + DihedralReportingStore& operator=(DihedralReportingStore&&) = delete; + + /** + * @brief Records a read and refuses the selected call before changing the buffer. + * @param startIndex First flat value index. + * @param buffer Receives the resident values on success. + * @return Injected error or the resident read result. + */ + Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + if(m_State.armed) + { + const usize call = ++m_State.reads[static_cast(m_Role)]; + m_State.calls.push_back({m_Role, false, startIndex, buffer.size()}); + if(m_State.failingRole == m_Role && !m_State.failingWrite && m_State.failingCall == call) + { + return MakeErrorResult(k_DihedralTransferError, DihedralFailureMessage(m_Role)); + } + } + return DataStore::copyIntoBuffer(startIndex, buffer); + } + + /** + * @brief Records writes and optionally cancels after the first completed output block. + * @param startIndex First flat value index. + * @param buffer Supplies values to publish unless the selected call fails. + * @return Injected error or the resident write result. + */ + Result<> copyFromBuffer(usize startIndex, nonstd::span buffer) override + { + if(!m_State.armed) + { + return DataStore::copyFromBuffer(startIndex, buffer); + } + const usize call = ++m_State.writes[static_cast(m_Role)]; + m_State.calls.push_back({m_Role, true, startIndex, buffer.size()}); + if(m_State.failingRole == m_Role && m_State.failingWrite && m_State.failingCall == call) + { + return MakeErrorResult(k_DihedralTransferError, DihedralFailureMessage(m_Role)); + } + auto result = DataStore::copyFromBuffer(startIndex, buffer); + if(result.valid() && m_Role == DihedralStoreRole::Angles && call == 1 && m_State.cancelAfterFirstWrite) + { + m_State.shouldCancel = true; + } + return result; + } + +private: + DihedralTransferState& m_State; + const DihedralStoreRole m_Role; +}; + +/** + * @struct DihedralTraceGuard + * @brief Limits recording to the target call and disarms on assertion failure. + */ +struct DihedralTraceGuard +{ + /** + * @brief Clears setup activity and starts the target transfer trace. + * @param state Outlives this guard and its reporting stores. + */ + explicit DihedralTraceGuard(DihedralTransferState& state) + : stateRef(state) + { + stateRef.calls.clear(); + stateRef.reads.fill(0); + stateRef.writes.fill(0); + stateRef.armed = true; + } + + /** + * @brief Stops recording even when the target or a test assertion throws. + */ + ~DihedralTraceGuard() + { + stateRef.armed = false; + } + + DihedralTraceGuard(const DihedralTraceGuard&) = delete; + DihedralTraceGuard(DihedralTraceGuard&&) = delete; + DihedralTraceGuard& operator=(const DihedralTraceGuard&) = delete; + DihedralTraceGuard& operator=(DihedralTraceGuard&&) = delete; + + DihedralTransferState& stateRef; +}; + +/** + * @struct DihedralFixture + * @brief Supplies eleven shared vertices and repeated triangles with a distinct final block tail. + */ +struct DihedralFixture +{ + DataStructure dataStructure; + TriangleGeom* geometryPtr = nullptr; + Float32Array* verticesPtr = nullptr; + DataArray* facesPtr = nullptr; + AttributeMatrix* faceDataPtr = nullptr; + AttributeMatrix* vertexDataPtr = nullptr; + + /** + * @brief Creates accepted filter inputs with no output array. + * @param faceCount Number of faces; zero retains a valid empty tuple shape. + * @param state Optional longer-lived state selects real resident reporting stores. + */ + explicit DihedralFixture(usize faceCount, DihedralTransferState* state = nullptr) + { + geometryPtr = TriangleGeom::Create(dataStructure, k_DihedralGeometryPath.getTargetName()); + REQUIRE(geometryPtr != nullptr); + faceDataPtr = AttributeMatrix::Create(dataStructure, "FaceData", {faceCount}, geometryPtr->getId()); + vertexDataPtr = AttributeMatrix::Create(dataStructure, "VertexData", {11}, geometryPtr->getId()); + REQUIRE(faceDataPtr != nullptr); + REQUIRE(vertexDataPtr != nullptr); + geometryPtr->setFaceAttributeMatrix(*faceDataPtr); + geometryPtr->setVertexAttributeMatrix(*vertexDataPtr); + std::shared_ptr> verticesStore; + std::shared_ptr> facesStore; + if(state != nullptr) + { + verticesStore = std::make_shared>(11, *state, DihedralStoreRole::Vertices); + facesStore = std::make_shared>(faceCount, *state, DihedralStoreRole::Faces); + } + else + { + verticesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_DihedralVerticesPath, {11}, {3}); + facesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_DihedralFacesPath, {faceCount}, {3}); + } + REQUIRE(verticesStore != nullptr); + REQUIRE(facesStore != nullptr); + verticesPtr = Float32Array::Create(dataStructure, "Vertices", verticesStore, geometryPtr->getId()); + facesPtr = DataArray::Create(dataStructure, "Faces", facesStore, geometryPtr->getId()); + REQUIRE(verticesPtr != nullptr); + REQUIRE(facesPtr != nullptr); + geometryPtr->setVertices(*verticesPtr); + geometryPtr->setFaceList(*facesPtr); + auto verticesResult = verticesStore->copyFromBuffer(0, nonstd::span(k_DihedralVertices.data(), k_DihedralVertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesResult); + std::vector buffer(k_DihedralBufferFaces * 3); + for(usize firstFace = 0; firstFace < faceCount; firstFace += k_DihedralBufferFaces) + { + const usize count = std::min(k_DihedralBufferFaces, faceCount - firstFace); + for(usize faceIndex = 0; faceIndex < count; ++faceIndex) + { + const auto& face = k_DihedralFaces[DihedralFixtureRow(firstFace + faceIndex)]; + std::copy(face.begin(), face.end(), buffer.begin() + faceIndex * 3); + } + auto result = facesStore->copyFromBuffer(firstFace * 3, nonstd::span(buffer.data(), count * 3)); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + } + + ~DihedralFixture() = default; + DihedralFixture(const DihedralFixture&) = default; + DihedralFixture(DihedralFixture&&) = default; + DihedralFixture& operator=(const DihedralFixture&) = default; + DihedralFixture& operator=(DihedralFixture&&) = default; + + Arguments arguments() const + { + Arguments args; + args.insertOrAssign(TriangleDihedralAngleFilter::k_TGeometryDataPath_Key, std::make_any(k_DihedralGeometryPath)); + args.insertOrAssign(TriangleDihedralAngleFilter::k_SurfaceMeshTriangleDihedralAnglesArrayName_Key, std::make_any("Angles")); + return args; + } + + Float64Array& output() + { + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_DihedralOutputPath)); + return dataStructure.getDataRefAs(k_DihedralOutputPath); + } +}; + +/** + * @brief Checks all input values and angles in bounded buffers, including an unpublished suffix. + * @param fixture Owns the input arrays and output. + * @param completedFaces Number of leading faces with computed angles; all remaining values retain the sentinel. + */ +void CheckDihedralFixture(DihedralFixture& fixture, usize completedFaces) +{ + auto& angles = fixture.output(); + const usize faceCount = fixture.facesPtr->getNumberOfTuples(); + REQUIRE(completedFaces <= faceCount); + REQUIRE(fixture.geometryPtr->getVertices() == fixture.verticesPtr); + REQUIRE(fixture.geometryPtr->getFaces() == fixture.facesPtr); + REQUIRE(fixture.geometryPtr->getFaceAttributeMatrix() == fixture.faceDataPtr); + REQUIRE(fixture.geometryPtr->getVertexAttributeMatrix() == fixture.vertexDataPtr); + REQUIRE(fixture.vertexDataPtr->getShape() == ShapeType{11}); + REQUIRE(fixture.faceDataPtr->getShape() == ShapeType{faceCount}); + REQUIRE(fixture.verticesPtr->getTupleShape() == ShapeType{11}); + REQUIRE(fixture.verticesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(fixture.facesPtr->getTupleShape() == ShapeType{faceCount}); + REQUIRE(fixture.facesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(angles.getTupleShape() == ShapeType{faceCount}); + REQUIRE(angles.getComponentShape() == ShapeType{1}); + REQUIRE(angles.getSize() == faceCount); + std::array vertices{}; + auto verticesResult = fixture.verticesPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(vertices.data(), vertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesResult); + REQUIRE(vertices == k_DihedralVertices); + std::vector faces(k_DihedralBufferFaces * 3); + std::vector values(k_DihedralBufferFaces); + for(usize firstFace = 0; firstFace < faceCount; firstFace += k_DihedralBufferFaces) + { + const usize count = std::min(k_DihedralBufferFaces, faceCount - firstFace); + auto facesResult = fixture.facesPtr->getDataStoreRef().copyIntoBuffer(firstFace * 3, nonstd::span(faces.data(), count * 3)); + auto anglesResult = angles.getDataStoreRef().copyIntoBuffer(firstFace, nonstd::span(values.data(), count)); + SIMPLNX_RESULT_REQUIRE_VALID(facesResult); + SIMPLNX_RESULT_REQUIRE_VALID(anglesResult); + for(usize localFace = 0; localFace < count; ++localFace) + { + const usize faceIndex = firstFace + localFace; + const usize row = DihedralFixtureRow(faceIndex); + for(usize component = 0; component < 3; ++component) + { + if(faces[localFace * 3 + component] != k_DihedralFaces[row][component]) + { + CAPTURE(faceIndex, component, faces[localFace * 3 + component], k_DihedralFaces[row][component]); + FAIL("Triangle dihedral input connectivity changed."); + } + } + if(!MatchesDihedralAngle(values[localFace], row, faceIndex < completedFaces)) + { + CAPTURE(faceIndex, row, values[localFace], completedFaces); + FAIL("Triangle dihedral angle differs from the independent fixture or unpublished sentinel."); + } + } + } + // Separate reads check the first face, block end, and distinct one-face tail. + for(const usize faceIndex : {usize{0}, k_DihedralBlockFaces - 1, k_DihedralBlockFaces}) + { + if(faceIndex >= faceCount) + { + continue; + } + float64 angle = k_DihedralSentinel; + auto result = angles.getDataStoreRef().copyIntoBuffer(faceIndex, nonstd::span(&angle, 1)); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CAPTURE(faceIndex, angle, completedFaces); + CHECK(MatchesDihedralAngle(angle, DihedralFixtureRow(faceIndex), faceIndex < completedFaces)); + } + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +/** + * @brief Validates filter inputs before supplying an output for the direct algorithm contract. + * @param fixture Owns the accepted geometry and receives the sentinel output. + * @param state Outlives the fixture and records the direct algorithm's transfers. + * @return The created resident output array. + */ +Float64Array* AddDihedralReportingOutput(DihedralFixture& fixture, DihedralTransferState& state) +{ + TriangleDihedralAngleFilter filter; + REQUIRE(fixture.dataStructure.getData(k_DihedralOutputPath) == nullptr); + const auto preflight = filter.preflight(fixture.dataStructure, fixture.arguments()); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + auto store = std::make_shared>(fixture.facesPtr->getNumberOfTuples(), state, DihedralStoreRole::Angles); + store->fill(k_DihedralSentinel); + auto* outputPtr = Float64Array::Create(fixture.dataStructure, "Angles", store, fixture.faceDataPtr->getId()); + REQUIRE(outputPtr != nullptr); + return outputPtr; +} +} // namespace + +TEST_CASE("SimplnxCore::TriangleDihedralAngleFilter: computation block tail", "[SimplnxCore][TriangleDihedralAngleFilter][D2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + DihedralFixture fixture(k_DihedralBlockFaces + 1); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + REQUIRE(fixture.dataStructure.getData(k_DihedralOutputPath) == nullptr); + TriangleDihedralAngleFilter filter; + const auto args = fixture.arguments(); + const auto preflight = filter.preflight(fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + const auto result = scope.executeFilter(filter, fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + scope.requireExpectedStore(fixture.output()); + CheckDihedralFixture(fixture, k_DihedralBlockFaces + 1); +} + +#if SIMPLNX_TEST_ALGORITHM_PATH != 2 +TEST_CASE("SimplnxCore::TriangleDihedralAngle: block transfer contracts", "[SimplnxCore][TriangleDihedralAngleFilter][D2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenarios({UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore}))); + const usize caseIndex = GENERATE(usize{0}, usize{1}, usize{2}, usize{3}, usize{4}, usize{5}, usize{6}); + struct ContractCase + { + const char* name; + DihedralStoreRole failingRole; + bool failingWrite; + usize failingCall; + usize traceLength; + usize completedFaces; + int32 contextCode; + const char* contextMessage; + }; + const std::array cases = { + {{"complete", DihedralStoreRole::Angles, false, 0, 5, k_DihedralBlockFaces + 1, 0, ""}, + {"vertex read", DihedralStoreRole::Vertices, false, 1, 1, 0, -65820, "Error reading vertex coordinates for triangle dihedral angle calculation."}, + {"first face read", DihedralStoreRole::Faces, false, 1, 2, 0, -65821, "Error reading connectivity for triangle dihedral angle calculation at triangle 0 (262144 triangles)."}, + {"tail face read", DihedralStoreRole::Faces, false, 2, 4, k_DihedralBlockFaces, -65821, "Error reading connectivity for triangle dihedral angle calculation at triangle 262144 (1 triangles)."}, + {"first output write", DihedralStoreRole::Angles, true, 1, 3, 0, -65822, "Error writing dihedral angles at triangle 0 (262144 triangles)."}, + {"tail output write", DihedralStoreRole::Angles, true, 2, 5, k_DihedralBlockFaces, -65822, "Error writing dihedral angles at triangle 262144 (1 triangles)."}, + {"cancel after first block", DihedralStoreRole::Angles, false, 0, 3, k_DihedralBlockFaces, 0, ""}}}; + const auto& contract = cases[caseIndex]; + CAPTURE(scenario, contract.name); + UnitTest::AlgorithmTestScope scope(scenario); + DihedralTransferState state; + DihedralFixture fixture(k_DihedralBlockFaces + 1, &state); + auto* outputPtr = AddDihedralReportingOutput(fixture, state); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + scope.requireExpectedStore(*outputPtr); + state.failingRole = contract.failingRole; + state.failingWrite = contract.failingWrite; + state.failingCall = contract.failingCall; + state.cancelAfterFirstWrite = caseIndex == 6; + TriangleDihedralAngleInputValues inputValues{k_DihedralGeometryPath, "Angles"}; + const IFilter::MessageHandler messages{}; + DihedralTraceGuard traceGuard(state); + const auto result = scope.execute([&] { return TriangleDihedralAngle(fixture.dataStructure, messages, state.shouldCancel, &inputValues)(); }); + const auto calls = state.calls; + state.armed = false; + REQUIRE(result.warnings().empty()); + if(contract.failingCall == 0) + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == k_DihedralTransferError); + CHECK(result.errors()[0].message == DihedralFailureMessage(contract.failingRole)); + CHECK(result.errors()[1].code == contract.contextCode); + CHECK(result.errors()[1].message == contract.contextMessage); + } + CHECK(state.shouldCancel.load() == state.cancelAfterFirstWrite); + const std::array expected = {{{DihedralStoreRole::Vertices, false, 0, 33}, + {DihedralStoreRole::Faces, false, 0, 786432}, + {DihedralStoreRole::Angles, true, 0, 262144}, + {DihedralStoreRole::Faces, false, 786432, 3}, + {DihedralStoreRole::Angles, true, 262144, 1}}}; + REQUIRE(calls.size() == contract.traceLength); + for(usize callIndex = 0; callIndex < calls.size(); ++callIndex) + { + CAPTURE(callIndex); + CHECK(calls[callIndex].role == expected[callIndex].role); + CHECK(calls[callIndex].write == expected[callIndex].write); + CHECK(calls[callIndex].start == expected[callIndex].start); + CHECK(calls[callIndex].count == expected[callIndex].count); + } + CheckDihedralFixture(fixture, contract.completedFaces); +} +#endif + +TEST_CASE("SimplnxCore::TriangleDihedralAngle: early exit controls", "[SimplnxCore][TriangleDihedralAngleFilter][D2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool preCancelled = GENERATE(false, true); + CAPTURE(scenario, preCancelled); + UnitTest::AlgorithmTestScope scope(scenario); + DihedralTransferState state; + DihedralFixture fixture(preCancelled ? 4 : 0, &state); + auto* outputPtr = AddDihedralReportingOutput(fixture, state); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + scope.requireExpectedStore(*outputPtr); + state.shouldCancel = preCancelled; + TriangleDihedralAngleInputValues inputValues{k_DihedralGeometryPath, "Angles"}; + const IFilter::MessageHandler messages{}; + DihedralTraceGuard traceGuard(state); + const auto result = scope.execute([&] { return TriangleDihedralAngle(fixture.dataStructure, messages, state.shouldCancel, &inputValues)(); }); + const auto calls = state.calls; + state.armed = false; + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.warnings().empty()); + CHECK(calls.empty()); + CHECK(state.shouldCancel.load() == preCancelled); + CheckDihedralFixture(fixture, 0); +} + +TEST_CASE("SimplnxCore::TriangleDihedralAngleFilter: empty geometry", "[SimplnxCore][TriangleDihedralAngleFilter][D2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + DihedralFixture fixture(0); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + REQUIRE(fixture.dataStructure.getData(k_DihedralOutputPath) == nullptr); + TriangleDihedralAngleFilter filter; + const auto args = fixture.arguments(); + const auto preflight = filter.preflight(fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + const auto result = scope.executeFilter(filter, fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + scope.requireExpectedStore(fixture.output()); + CheckDihedralFixture(fixture, 0); +} + +#if SIMPLNX_TEST_ALGORITHM_PATH != 2 +TEST_CASE("SimplnxCore::TriangleDihedralAngle: buffered invalid connectivity", "[SimplnxCore][TriangleDihedralAngleFilter][D2][InvalidConnectivity]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenarios({UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore}))); + const auto invalidVertex = GENERATE(IGeometry::MeshIndexType{3}, std::numeric_limits::max()); + const usize invalidFace = GENERATE(usize{0}, usize{1}); + const usize invalidCorner = GENERATE(usize{0}, usize{1}, usize{2}); + CAPTURE(scenario, invalidVertex, invalidFace, invalidCorner); + UnitTest::AlgorithmTestScope scope(scenario); + DihedralTransferState state; + DihedralFixture fixture(2, &state); + auto resizeVertices = fixture.geometryPtr->resizeVertexList(3); + auto resizeVertexData = fixture.vertexDataPtr->resizeTuples({3}); + SIMPLNX_RESULT_REQUIRE_VALID(resizeVertices); + SIMPLNX_RESULT_REQUIRE_VALID(resizeVertexData); + // Three vertices admit only IDs 0, 1, and 2. Every corner must fail before offset arithmetic. + std::array faces = {0, 1, 2, 0, 1, 2}; + faces.at((invalidFace * 3) + invalidCorner) = invalidVertex; + auto facesWrite = fixture.facesPtr->getDataStoreRef().copyFromBuffer(0, nonstd::span(faces.data(), faces.size())); + SIMPLNX_RESULT_REQUIRE_VALID(facesWrite); + auto* outputPtr = AddDihedralReportingOutput(fixture, state); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + scope.requireExpectedStore(*outputPtr); + TriangleDihedralAngleInputValues inputValues{.InputTriangleGeometryPath = k_DihedralGeometryPath, .SurfaceMeshTriangleDihedralAnglesArrayName = "Angles"}; + const IFilter::MessageHandler messages{}; + const DihedralTraceGuard traceGuard(state); + const auto result = scope.execute([&] { return TriangleDihedralAngle(fixture.dataStructure, messages, state.shouldCancel, &inputValues)(); }); + state.armed = false; + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -65823); + CHECK(result.errors()[0].message == fmt::format("Triangle {} references vertex index {}, but the vertex tuple count is 3. Valid vertex indices are less than 3.", invalidFace, invalidVertex)); + CHECK(result.warnings().empty()); + CHECK(state.writes[static_cast(DihedralStoreRole::Angles)] == 0); + std::array actual{}; + auto outputRead = outputPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actual.data(), actual.size())); + SIMPLNX_RESULT_REQUIRE_VALID(outputRead); + for(const float64 value : actual) + { + CHECK(value == k_DihedralSentinel); + } + std::array actualFaces{}; + auto facesRead = fixture.facesPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualFaces.data(), actualFaces.size())); + SIMPLNX_RESULT_REQUIRE_VALID(facesRead); + CHECK(actualFaces == faces); + std::array actualVertices{}; + auto verticesRead = fixture.verticesPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualVertices.data(), actualVertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesRead); + CHECK(std::equal(actualVertices.begin(), actualVertices.end(), k_DihedralVertices.begin())); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} +#endif \ No newline at end of file diff --git a/src/Plugins/SimplnxCore/test/TriangleNormalFilterTest.cpp b/src/Plugins/SimplnxCore/test/TriangleNormalFilterTest.cpp index 78dc89dd5a..0543aad227 100644 --- a/src/Plugins/SimplnxCore/test/TriangleNormalFilterTest.cpp +++ b/src/Plugins/SimplnxCore/test/TriangleNormalFilterTest.cpp @@ -1,8 +1,12 @@ #include "SimplnxCore/Filters/TriangleNormalFilter.hpp" +#include "SimplnxCore/Filters/Algorithms/TriangleNormal.hpp" #include "SimplnxCore/Filters/ReadStlFileFilter.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" #include "simplnx/Core/Application.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" #include "simplnx/DataStructure/IO/HDF5/DataStructureWriter.hpp" #include "simplnx/Parameters/ArrayCreationParameter.hpp" @@ -10,9 +14,21 @@ #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" #include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" #include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -30,6 +46,9 @@ constexpr float64 k_max_difference = 0.0001; TEST_CASE("SimplnxCore::TriangleNormalFilter", "[SimplnxCore][TriangleNormalFilter]") { UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); std::string triangleGeometryName = "[Triangle Geometry]"; std::string triangleFaceDataGroupName = "Face Data"; @@ -70,6 +89,12 @@ TEST_CASE("SimplnxCore::TriangleNormalFilter", "[SimplnxCore][TriangleNormalFilt std::string triangleNormalsName = "Face Normals (computed)"; DataPath geometryPath = DataPath({triangleGeometryName}); + const DataPath verticesPath = geometryPath.createChildPath(INodeGeometry0D::k_SharedVertexListName); + const DataPath facesPath = geometryPath.createChildPath(INodeGeometry2D::k_SharedFacesListName); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(verticesPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(facesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(verticesPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(facesPath)); // Create default Parameters for the filter. args.insertOrAssign(TriangleNormalFilter::k_TriGeometryDataPath_Key, std::make_any(geometryPath)); @@ -80,7 +105,7 @@ TEST_CASE("SimplnxCore::TriangleNormalFilter", "[SimplnxCore][TriangleNormalFilt SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); // Execute the filter and check the result - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); // Write the DataStructure out to the file system @@ -89,6 +114,8 @@ TEST_CASE("SimplnxCore::TriangleNormalFilter", "[SimplnxCore][TriangleNormalFilt #endif DataPath triangleNormalsDataPath = geometryPath.createChildPath(triangleFaceDataGroupName).createChildPath(triangleNormalsName); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(triangleNormalsDataPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(triangleNormalsDataPath)); // Let's compare the normals. DataPath normalsDataPath({triangleGeometryName, triangleFaceDataGroupName, normalsDataArrayName}); @@ -143,3 +170,554 @@ TEST_CASE("SimplnxCore::TriangleNormalFilter: SIMPL Backwards Compatibility", "[ } } } + +namespace +{ +constexpr usize k_NormalBlockFaces = 262144; +constexpr usize k_NormalBufferFaces = 4096; +constexpr float64 k_NormalSentinel = -37.0; +constexpr int32 k_NormalTransferError = -98761; +const DataPath k_NormalGeometryPath({"NormalMesh"}); +const DataPath k_NormalVerticesPath = k_NormalGeometryPath.createChildPath("Vertices"); +const DataPath k_NormalFacesPath = k_NormalGeometryPath.createChildPath("Faces"); +const DataPath k_NormalOutputPath = k_NormalGeometryPath.createChildPath("FaceData").createChildPath("Normals"); +constexpr std::array k_NormalVertices = {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 1, 0, 2, 1}; +constexpr std::array, 5> k_NormalFaces = {{{0, 1, 2}, {0, 2, 1}, {0, 2, 3}, {0, 4, 5}, {0, 3, 2}}}; +// The sloped face has cross product (-2, -3, 6) and length 7. These literals require double arithmetic. +constexpr std::array, 5> k_ExpectedNormals = { + {{0, 0, 1}, {0, 0, -1}, {1, 0, 0}, {-0.2857142857142857142857142857, -0.4285714285714285714285714286, 0.8571428571428571428571428571}, {-1, 0, 0}}}; + +usize NormalFixtureRow(usize faceIndex) +{ + return faceIndex == k_NormalBlockFaces ? 4 : faceIndex % 4; +} + +enum class NormalStoreRole : usize +{ + Vertices, + Faces, + Normals +}; + +struct NormalTransfer +{ + NormalStoreRole role; + bool write; + usize start; + usize count; +}; + +struct NormalTransferState +{ + bool armed = false; + NormalStoreRole failingRole = NormalStoreRole::Normals; + bool failingWrite = false; + usize failingCall = 0; + bool cancelAfterFirstWrite = false; + std::atomic_bool shouldCancel{false}; + std::array reads{}; + std::array writes{}; + std::vector calls; +}; + +const char* NormalFailureMessage(NormalStoreRole role) +{ + switch(role) + { + case NormalStoreRole::Vertices: + return "Injected triangle normal vertex read failure."; + case NormalStoreRole::Faces: + return "Injected triangle normal connectivity read failure."; + case NormalStoreRole::Normals: + return "Injected triangle normal output write failure."; + } + return "Unknown triangle normal transfer failure."; +} + +/** + * @class NormalReportingStore + * @brief Records sequential bulk transfers and rejects selected calls before touching resident values. + * @tparam T Stored scalar type. + * @note The borrowed state outlives the store. This fixture does not model partial backend writes. + */ +template +class NormalReportingStore final : public DataStore +{ +public: + /** + * @brief Creates real resident storage with a disarmed, longer-lived trace. + * @param tuples Number of three-component tuples. + * @param state Records target transfers only. + * @param role Identifies the array in each transfer. + */ + NormalReportingStore(usize tuples, NormalTransferState& state, NormalStoreRole role) + : DataStore(ShapeType{tuples}, ShapeType{3}, T{}) + , m_State(state) + , m_Role(role) + { + } + + ~NormalReportingStore() override = default; + NormalReportingStore(const NormalReportingStore&) = default; + NormalReportingStore(NormalReportingStore&&) = default; + NormalReportingStore& operator=(const NormalReportingStore&) = delete; + NormalReportingStore& operator=(NormalReportingStore&&) = delete; + + /** + * @brief Records a read and refuses the selected call before changing the buffer. + * @param startIndex First flat value index. + * @param buffer Receives the resident values on success. + * @return Injected error or the resident read result. + */ + Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + if(m_State.armed) + { + const usize call = ++m_State.reads[static_cast(m_Role)]; + m_State.calls.push_back({m_Role, false, startIndex, buffer.size()}); + if(m_State.failingRole == m_Role && !m_State.failingWrite && m_State.failingCall == call) + { + return MakeErrorResult(k_NormalTransferError, NormalFailureMessage(m_Role)); + } + } + return DataStore::copyIntoBuffer(startIndex, buffer); + } + + /** + * @brief Records writes and optionally cancels after the first completed output block. + * @param startIndex First flat value index. + * @param buffer Supplies values to publish unless the selected call fails. + * @return Injected error or the resident write result. + */ + Result<> copyFromBuffer(usize startIndex, nonstd::span buffer) override + { + if(!m_State.armed) + { + return DataStore::copyFromBuffer(startIndex, buffer); + } + const usize call = ++m_State.writes[static_cast(m_Role)]; + m_State.calls.push_back({m_Role, true, startIndex, buffer.size()}); + if(m_State.failingRole == m_Role && m_State.failingWrite && m_State.failingCall == call) + { + return MakeErrorResult(k_NormalTransferError, NormalFailureMessage(m_Role)); + } + auto result = DataStore::copyFromBuffer(startIndex, buffer); + if(result.valid() && m_Role == NormalStoreRole::Normals && call == 1 && m_State.cancelAfterFirstWrite) + { + m_State.shouldCancel = true; + } + return result; + } + +private: + NormalTransferState& m_State; + const NormalStoreRole m_Role; +}; + +/** + * @struct NormalTraceGuard + * @brief Limits recording to the target call and disarms on assertion failure. + */ +struct NormalTraceGuard +{ + /** + * @brief Clears setup activity and starts the target transfer trace. + * @param state Outlives this guard and its reporting stores. + */ + explicit NormalTraceGuard(NormalTransferState& state) + : stateRef(state) + { + stateRef.calls.clear(); + stateRef.reads.fill(0); + stateRef.writes.fill(0); + stateRef.armed = true; + } + + /** + * @brief Stops recording even when the target or a test assertion throws. + */ + ~NormalTraceGuard() + { + stateRef.armed = false; + } + + NormalTraceGuard(const NormalTraceGuard&) = delete; + NormalTraceGuard(NormalTraceGuard&&) = delete; + NormalTraceGuard& operator=(const NormalTraceGuard&) = delete; + NormalTraceGuard& operator=(NormalTraceGuard&&) = delete; + + NormalTransferState& stateRef; +}; + +/** + * @struct NormalFixture + * @brief Supplies six shared vertices and repeated triangles with a distinct final block tail. + */ +struct NormalFixture +{ + DataStructure dataStructure; + TriangleGeom* geometryPtr = nullptr; + Float32Array* verticesPtr = nullptr; + DataArray* facesPtr = nullptr; + AttributeMatrix* faceDataPtr = nullptr; + AttributeMatrix* vertexDataPtr = nullptr; + + /** + * @brief Creates accepted filter inputs with no output array. + * @param faceCount Number of faces; zero retains a valid empty tuple shape. + * @param state Optional longer-lived state selects real resident reporting stores. + */ + explicit NormalFixture(usize faceCount, NormalTransferState* state = nullptr) + { + geometryPtr = TriangleGeom::Create(dataStructure, k_NormalGeometryPath.getTargetName()); + REQUIRE(geometryPtr != nullptr); + faceDataPtr = AttributeMatrix::Create(dataStructure, "FaceData", {faceCount}, geometryPtr->getId()); + vertexDataPtr = AttributeMatrix::Create(dataStructure, "VertexData", {6}, geometryPtr->getId()); + REQUIRE(faceDataPtr != nullptr); + REQUIRE(vertexDataPtr != nullptr); + geometryPtr->setFaceAttributeMatrix(*faceDataPtr); + geometryPtr->setVertexAttributeMatrix(*vertexDataPtr); + std::shared_ptr> verticesStore; + std::shared_ptr> facesStore; + if(state != nullptr) + { + verticesStore = std::make_shared>(6, *state, NormalStoreRole::Vertices); + facesStore = std::make_shared>(faceCount, *state, NormalStoreRole::Faces); + } + else + { + verticesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_NormalVerticesPath, {6}, {3}); + facesStore = DataStoreUtilities::CreateDataStore(dataStructure, k_NormalFacesPath, {faceCount}, {3}); + } + REQUIRE(verticesStore != nullptr); + REQUIRE(facesStore != nullptr); + verticesPtr = Float32Array::Create(dataStructure, "Vertices", verticesStore, geometryPtr->getId()); + facesPtr = DataArray::Create(dataStructure, "Faces", facesStore, geometryPtr->getId()); + REQUIRE(verticesPtr != nullptr); + REQUIRE(facesPtr != nullptr); + geometryPtr->setVertices(*verticesPtr); + geometryPtr->setFaceList(*facesPtr); + auto verticesResult = verticesStore->copyFromBuffer(0, nonstd::span(k_NormalVertices.data(), k_NormalVertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesResult); + std::vector buffer(k_NormalBufferFaces * 3); + for(usize firstFace = 0; firstFace < faceCount; firstFace += k_NormalBufferFaces) + { + const usize count = std::min(k_NormalBufferFaces, faceCount - firstFace); + for(usize faceIndex = 0; faceIndex < count; ++faceIndex) + { + const auto& face = k_NormalFaces[NormalFixtureRow(firstFace + faceIndex)]; + std::copy(face.begin(), face.end(), buffer.begin() + faceIndex * 3); + } + auto result = facesStore->copyFromBuffer(firstFace * 3, nonstd::span(buffer.data(), count * 3)); + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + } + + ~NormalFixture() = default; + NormalFixture(const NormalFixture&) = default; + NormalFixture(NormalFixture&&) = default; + NormalFixture& operator=(const NormalFixture&) = default; + NormalFixture& operator=(NormalFixture&&) = default; + + Arguments arguments() const + { + Arguments args; + args.insertOrAssign(TriangleNormalFilter::k_TriGeometryDataPath_Key, std::make_any(k_NormalGeometryPath)); + args.insertOrAssign(TriangleNormalFilter::k_SurfaceMeshTriangleNormalsArrayName_Key, std::make_any("Normals")); + return args; + } + + Float64Array& output() + { + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_NormalOutputPath)); + return dataStructure.getDataRefAs(k_NormalOutputPath); + } +}; + +/** + * @brief Checks every input and output value with bounded buffers, including an unpublished suffix. + * @param fixture Owns the input arrays and output. + * @param completedFaces Number of leading faces with computed normals; all remaining values retain the sentinel. + */ +void CheckNormalFixture(NormalFixture& fixture, usize completedFaces) +{ + auto& normals = fixture.output(); + const usize faceCount = fixture.facesPtr->getNumberOfTuples(); + REQUIRE(completedFaces <= faceCount); + REQUIRE(fixture.geometryPtr->getVertices() == fixture.verticesPtr); + REQUIRE(fixture.geometryPtr->getFaces() == fixture.facesPtr); + REQUIRE(fixture.geometryPtr->getFaceAttributeMatrix() == fixture.faceDataPtr); + REQUIRE(fixture.geometryPtr->getVertexAttributeMatrix() == fixture.vertexDataPtr); + REQUIRE(fixture.vertexDataPtr->getShape() == ShapeType{6}); + REQUIRE(fixture.faceDataPtr->getShape() == ShapeType{faceCount}); + REQUIRE(fixture.verticesPtr->getTupleShape() == ShapeType{6}); + REQUIRE(fixture.verticesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(fixture.facesPtr->getTupleShape() == ShapeType{faceCount}); + REQUIRE(fixture.facesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(normals.getTupleShape() == ShapeType{faceCount}); + REQUIRE(normals.getComponentShape() == ShapeType{3}); + REQUIRE(normals.getSize() == faceCount * 3); + std::array vertices{}; + auto verticesResult = fixture.verticesPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(vertices.data(), vertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesResult); + REQUIRE(vertices == k_NormalVertices); + std::vector faces(k_NormalBufferFaces * 3); + std::vector values(k_NormalBufferFaces * 3); + for(usize firstFace = 0; firstFace < faceCount; firstFace += k_NormalBufferFaces) + { + const usize count = std::min(k_NormalBufferFaces, faceCount - firstFace); + auto facesResult = fixture.facesPtr->getDataStoreRef().copyIntoBuffer(firstFace * 3, nonstd::span(faces.data(), count * 3)); + auto normalsResult = normals.getDataStoreRef().copyIntoBuffer(firstFace * 3, nonstd::span(values.data(), count * 3)); + SIMPLNX_RESULT_REQUIRE_VALID(facesResult); + SIMPLNX_RESULT_REQUIRE_VALID(normalsResult); + for(usize valueIndex = 0; valueIndex < count * 3; ++valueIndex) + { + const usize faceIndex = firstFace + valueIndex / 3; + const usize component = valueIndex % 3; + const usize row = NormalFixtureRow(faceIndex); + const float64 expected = faceIndex < completedFaces ? k_ExpectedNormals[row][component] : k_NormalSentinel; + if(faces[valueIndex] != k_NormalFaces[row][component] || !(std::abs(values[valueIndex] - expected) <= 1.0e-14)) + { + CAPTURE(faceIndex, component, faces[valueIndex], k_NormalFaces[row][component], values[valueIndex], expected); + FAIL("Triangle normal values or input connectivity differ from the independent fixture."); + } + } + } + // These separate reads make the first face, block end, and one-face tail explicit witnesses. + for(const usize faceIndex : {usize{0}, k_NormalBlockFaces - 1, k_NormalBlockFaces}) + { + if(faceIndex >= faceCount) + { + continue; + } + std::array normal{}; + auto result = normals.getDataStoreRef().copyIntoBuffer(faceIndex * 3, nonstd::span(normal.data(), normal.size())); + SIMPLNX_RESULT_REQUIRE_VALID(result); + for(usize component = 0; component < 3; ++component) + { + const float64 expected = faceIndex < completedFaces ? k_ExpectedNormals[NormalFixtureRow(faceIndex)][component] : k_NormalSentinel; + CAPTURE(faceIndex, component); + CHECK(std::abs(normal[component] - expected) <= 1.0e-14); + } + } + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +/** + * @brief Validates filter inputs before supplying an output for the direct algorithm contract. + * @param fixture Owns the accepted geometry and receives the sentinel output. + * @param state Outlives the fixture and records the direct algorithm's transfers. + * @return The created resident output array. + */ +Float64Array* AddNormalReportingOutput(NormalFixture& fixture, NormalTransferState& state) +{ + TriangleNormalFilter filter; + REQUIRE(fixture.dataStructure.getData(k_NormalOutputPath) == nullptr); + const auto preflight = filter.preflight(fixture.dataStructure, fixture.arguments()); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + auto store = std::make_shared>(fixture.facesPtr->getNumberOfTuples(), state, NormalStoreRole::Normals); + store->fill(k_NormalSentinel); + auto* outputPtr = Float64Array::Create(fixture.dataStructure, "Normals", store, fixture.faceDataPtr->getId()); + REQUIRE(outputPtr != nullptr); + return outputPtr; +} +} // namespace + +TEST_CASE("SimplnxCore::TriangleNormalFilter: computation block tail", "[SimplnxCore][TriangleNormalFilter][N2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + NormalFixture fixture(k_NormalBlockFaces + 1); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + REQUIRE(fixture.dataStructure.getData(k_NormalOutputPath) == nullptr); + TriangleNormalFilter filter; + const auto args = fixture.arguments(); + const auto preflight = filter.preflight(fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + const auto result = scope.executeFilter(filter, fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + scope.requireExpectedStore(fixture.output()); + CheckNormalFixture(fixture, k_NormalBlockFaces + 1); +} + +#if SIMPLNX_TEST_ALGORITHM_PATH != 2 +TEST_CASE("SimplnxCore::TriangleNormal: block transfer contracts", "[SimplnxCore][TriangleNormalFilter][N2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenarios({UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore}))); + const usize caseIndex = GENERATE(usize{0}, usize{1}, usize{2}, usize{3}, usize{4}, usize{5}, usize{6}); + struct ContractCase + { + const char* name; + NormalStoreRole failingRole; + bool failingWrite; + usize failingCall; + usize traceLength; + usize completedFaces; + int32 contextCode; + const char* contextMessage; + }; + const std::array cases = { + {{"complete", NormalStoreRole::Normals, false, 0, 5, k_NormalBlockFaces + 1, 0, ""}, + {"vertex read", NormalStoreRole::Vertices, false, 1, 1, 0, -65810, "Error reading vertex coordinates for triangle normal calculation."}, + {"first face read", NormalStoreRole::Faces, false, 1, 2, 0, -65811, "Error reading connectivity for triangle normal calculation at triangle 0 (262144 triangles)."}, + {"tail face read", NormalStoreRole::Faces, false, 2, 4, k_NormalBlockFaces, -65811, "Error reading connectivity for triangle normal calculation at triangle 262144 (1 triangles)."}, + {"first output write", NormalStoreRole::Normals, true, 1, 3, 0, -65812, "Error writing normals at triangle 0 (262144 triangles)."}, + {"tail output write", NormalStoreRole::Normals, true, 2, 5, k_NormalBlockFaces, -65812, "Error writing normals at triangle 262144 (1 triangles)."}, + {"cancel after first block", NormalStoreRole::Normals, false, 0, 3, k_NormalBlockFaces, 0, ""}}}; + const auto& contract = cases[caseIndex]; + CAPTURE(scenario, contract.name); + UnitTest::AlgorithmTestScope scope(scenario); + NormalTransferState state; + NormalFixture fixture(k_NormalBlockFaces + 1, &state); + auto* outputPtr = AddNormalReportingOutput(fixture, state); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + scope.requireExpectedStore(*outputPtr); + state.failingRole = contract.failingRole; + state.failingWrite = contract.failingWrite; + state.failingCall = contract.failingCall; + state.cancelAfterFirstWrite = caseIndex == 6; + TriangleNormalInputValues inputValues{k_NormalGeometryPath, "Normals"}; + const IFilter::MessageHandler messages{}; + NormalTraceGuard traceGuard(state); + const auto result = scope.execute([&] { return TriangleNormal(fixture.dataStructure, messages, state.shouldCancel, &inputValues)(); }); + const auto calls = state.calls; + state.armed = false; + REQUIRE(result.warnings().empty()); + if(contract.failingCall == 0) + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == k_NormalTransferError); + CHECK(result.errors()[0].message == NormalFailureMessage(contract.failingRole)); + CHECK(result.errors()[1].code == contract.contextCode); + CHECK(result.errors()[1].message == contract.contextMessage); + } + CHECK(state.shouldCancel.load() == state.cancelAfterFirstWrite); + const std::array expected = {{{NormalStoreRole::Vertices, false, 0, 18}, + {NormalStoreRole::Faces, false, 0, 786432}, + {NormalStoreRole::Normals, true, 0, 786432}, + {NormalStoreRole::Faces, false, 786432, 3}, + {NormalStoreRole::Normals, true, 786432, 3}}}; + REQUIRE(calls.size() == contract.traceLength); + for(usize callIndex = 0; callIndex < calls.size(); ++callIndex) + { + CAPTURE(callIndex); + CHECK(calls[callIndex].role == expected[callIndex].role); + CHECK(calls[callIndex].write == expected[callIndex].write); + CHECK(calls[callIndex].start == expected[callIndex].start); + CHECK(calls[callIndex].count == expected[callIndex].count); + } + CheckNormalFixture(fixture, contract.completedFaces); +} +#endif + +TEST_CASE("SimplnxCore::TriangleNormal: early exit controls", "[SimplnxCore][TriangleNormalFilter][N2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool preCancelled = GENERATE(false, true); + CAPTURE(scenario, preCancelled); + UnitTest::AlgorithmTestScope scope(scenario); + NormalTransferState state; + NormalFixture fixture(preCancelled ? 4 : 0, &state); + auto* outputPtr = AddNormalReportingOutput(fixture, state); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + scope.requireExpectedStore(*outputPtr); + state.shouldCancel = preCancelled; + TriangleNormalInputValues inputValues{k_NormalGeometryPath, "Normals"}; + const IFilter::MessageHandler messages{}; + NormalTraceGuard traceGuard(state); + const auto result = scope.execute([&] { return TriangleNormal(fixture.dataStructure, messages, state.shouldCancel, &inputValues)(); }); + const auto calls = state.calls; + state.armed = false; + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.warnings().empty()); + CHECK(calls.empty()); + CHECK(state.shouldCancel.load() == preCancelled); + CheckNormalFixture(fixture, 0); +} + +TEST_CASE("SimplnxCore::TriangleNormalFilter: empty geometry", "[SimplnxCore][TriangleNormalFilter][N2]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + NormalFixture fixture(0); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + REQUIRE(fixture.dataStructure.getData(k_NormalOutputPath) == nullptr); + TriangleNormalFilter filter; + const auto args = fixture.arguments(); + const auto preflight = filter.preflight(fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflight.outputActions); + const auto result = scope.executeFilter(filter, fixture.dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(result.result); + scope.requireExpectedStore(fixture.output()); + CheckNormalFixture(fixture, 0); +} + +#if SIMPLNX_TEST_ALGORITHM_PATH != 2 +TEST_CASE("SimplnxCore::TriangleNormal: buffered invalid connectivity", "[SimplnxCore][TriangleNormalFilter][N2][InvalidConnectivity]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenarios({UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore}))); + const auto invalidVertex = GENERATE(IGeometry::MeshIndexType{3}, std::numeric_limits::max()); + const usize invalidFace = GENERATE(usize{0}, usize{1}); + const usize invalidCorner = GENERATE(usize{0}, usize{1}, usize{2}); + CAPTURE(scenario, invalidVertex, invalidFace, invalidCorner); + UnitTest::AlgorithmTestScope scope(scenario); + NormalTransferState state; + NormalFixture fixture(2, &state); + auto resizeVertices = fixture.geometryPtr->resizeVertexList(3); + auto resizeVertexData = fixture.vertexDataPtr->resizeTuples({3}); + SIMPLNX_RESULT_REQUIRE_VALID(resizeVertices); + SIMPLNX_RESULT_REQUIRE_VALID(resizeVertexData); + // Three vertices admit only IDs 0, 1, and 2. Every corner must fail before offset arithmetic. + std::array faces = {0, 1, 2, 0, 1, 2}; + faces.at((invalidFace * 3) + invalidCorner) = invalidVertex; + auto facesWrite = fixture.facesPtr->getDataStoreRef().copyFromBuffer(0, nonstd::span(faces.data(), faces.size())); + SIMPLNX_RESULT_REQUIRE_VALID(facesWrite); + auto* outputPtr = AddNormalReportingOutput(fixture, state); + scope.requireExpectedStore(*fixture.verticesPtr); + scope.requireExpectedStore(*fixture.facesPtr); + scope.requireExpectedStore(*outputPtr); + TriangleNormalInputValues inputValues{.InputTriangleGeometryPath = k_NormalGeometryPath, .OutputNormalsArrayName = "Normals"}; + const IFilter::MessageHandler messages{}; + const NormalTraceGuard traceGuard(state); + const auto result = scope.execute([&] { return TriangleNormal(fixture.dataStructure, messages, state.shouldCancel, &inputValues)(); }); + state.armed = false; + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -65813); + CHECK(result.errors()[0].message == fmt::format("Triangle {} references vertex index {}, but the vertex tuple count is 3. Valid vertex indices are less than 3.", invalidFace, invalidVertex)); + CHECK(result.warnings().empty()); + CHECK(state.writes[static_cast(NormalStoreRole::Normals)] == 0); + std::array actual{}; + auto outputRead = outputPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actual.data(), actual.size())); + SIMPLNX_RESULT_REQUIRE_VALID(outputRead); + for(const float64 value : actual) + { + CHECK(value == k_NormalSentinel); + } + std::array actualFaces{}; + auto facesRead = fixture.facesPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualFaces.data(), actualFaces.size())); + SIMPLNX_RESULT_REQUIRE_VALID(facesRead); + CHECK(actualFaces == faces); + std::array actualVertices{}; + auto verticesRead = fixture.verticesPtr->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualVertices.data(), actualVertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesRead); + CHECK(std::equal(actualVertices.begin(), actualVertices.end(), k_NormalVertices.begin())); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} +#endif \ No newline at end of file diff --git a/src/Plugins/SimplnxCore/test/VerifyTriangleWindingTest.cpp b/src/Plugins/SimplnxCore/test/VerifyTriangleWindingTest.cpp index 7f1d11bcce..e82c2a20c7 100644 --- a/src/Plugins/SimplnxCore/test/VerifyTriangleWindingTest.cpp +++ b/src/Plugins/SimplnxCore/test/VerifyTriangleWindingTest.cpp @@ -6,11 +6,22 @@ #include "SimplnxCore/SimplnxCore_test_dirs.hpp" #include "simplnx/Core/Application.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" #include "simplnx/Parameters/DataGroupSelectionParameter.hpp" #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/Pipeline/PipelineFilter.hpp" +#include "simplnx/UnitTest/AlgorithmTestScope.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" + +#include +#include +#include +#include +#include namespace fs = std::filesystem; using namespace nx::core; @@ -19,10 +30,70 @@ namespace { const DataPath k_TriangleGeomPath = DataPath({Constants::k_DataContainer}); const DataPath k_FaceDataPath = k_TriangleGeomPath.createChildPath(Constants::k_Face_Data); + +/** + * @brief Creates four resident tuples without consulting storage preferences. + * @tparam T Value type of the fixture array. + * @param dataStructure Owns the new array and its existing parent. + * @param path Identifies the array below an existing geometry or attribute matrix. + * @param componentCount Number of values per tuple. + * @param values Supplies all four tuples. + * @return Array owned by dataStructure. + */ +template +DataArray* CreateDisabledNormalsArray(DataStructure& dataStructure, const DataPath& path, usize componentCount, nonstd::span values) +{ + REQUIRE(values.size() == 4 * componentCount); + const auto* parentPtr = dataStructure.getData(path.getParent()); + REQUIRE(parentPtr != nullptr); + auto store = std::make_shared>(ShapeType{4}, ShapeType{componentCount}, T{}); + auto writeResult = store->copyFromBuffer(0, values); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + auto* arrayPtr = DataArray::Create(dataStructure, path.getTargetName(), store, parentPtr->getId()); + REQUIRE(arrayPtr != nullptr); + return arrayPtr; +} + +/** + * @brief Checks every value, shape, and identity while the DataStructure owns the array. + * @tparam T Value type of the fixture array. + * @param dataStructure Owns the array under test. + * @param path Identifies the original array location. + * @param expectedId Original DataObject ID. + * @param expectedStorePtr Original store address, used only for comparison. + * @param componentCount Expected values per tuple. + * @param expectedValues Independent expected values for all four tuples. + */ +template +void CheckDisabledNormalsArray(const DataStructure& dataStructure, const DataPath& path, DataObject::IdType expectedId, const AbstractDataStore* expectedStorePtr, usize componentCount, + nonstd::span expectedValues) +{ + REQUIRE_NOTHROW(dataStructure.getDataRefAs>(path)); + const auto& array = dataStructure.getDataRefAs>(path); + REQUIRE(array.getId() == expectedId); + REQUIRE(array.getDataStore() == expectedStorePtr); + REQUIRE(array.getDataStoreRef().getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(array.getTupleShape() == ShapeType{4}); + REQUIRE(array.getComponentShape() == ShapeType{componentCount}); + REQUIRE(array.getSize() == expectedValues.size()); + std::vector actualValues(expectedValues.size(), T{}); + auto readResult = array.getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualValues.data(), actualValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(readResult); + for(usize valueIndex = 0; valueIndex < expectedValues.size(); ++valueIndex) + { + CAPTURE(path.toString(), valueIndex); + CHECK(actualValues[valueIndex] == expectedValues[valueIndex]); + } +} } // namespace TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: Valid Face Labels Execution", "[SimplnxCore][VerifyTriangleWindingFilter]") { + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_TestFilesDir, "STL_Models.tar.gz", "STL_Models"); DataStructure dataStructure = {}; @@ -102,13 +173,37 @@ TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: Valid Face Labels Execution args.insertOrAssign(VerifyTriangleWindingFilter::k_RepairNormals_Key, std::make_any(true)); args.insertOrAssign(VerifyTriangleWindingFilter::k_TriangleNormalsPath_Key, std::make_any(k_FaceDataPath.createChildPath(Constants::k_FaceNormals))); + const DataPath labelsPath = k_FaceDataPath.createChildPath(Constants::k_FaceLabels); + const DataPath normalsPath = k_FaceDataPath.createChildPath(Constants::k_FaceNormals); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_TriangleGeomPath)); + const auto& triangleGeom = dataStructure.getDataRefAs(k_TriangleGeomPath); + REQUIRE(triangleGeom.getVertices() != nullptr); + REQUIRE(triangleGeom.getFaces() != nullptr); + scope.requireExpectedStore(*triangleGeom.getVertices()); + scope.requireExpectedStore(*triangleGeom.getFaces()); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(labelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(normalsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(labelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(normalsPath)); + // Preflight the filter and check result auto preflightResult = filter.preflight(dataStructure, args); REQUIRE(preflightResult.outputActions.valid()); // Execute the filter and check the result - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); REQUIRE(executeResult.result.valid()); + + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_TriangleGeomPath)); + const auto& repairedTriangleGeom = dataStructure.getDataRefAs(k_TriangleGeomPath); + REQUIRE(repairedTriangleGeom.getVertices() != nullptr); + REQUIRE(repairedTriangleGeom.getFaces() != nullptr); + scope.requireExpectedStore(*repairedTriangleGeom.getVertices()); + scope.requireExpectedStore(*repairedTriangleGeom.getFaces()); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(labelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(normalsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(labelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(normalsPath)); } DataStructure exemplarDataStructure = {}; @@ -145,6 +240,11 @@ TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: Valid Face Labels Execution TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: Valid Region Ids Execution", "[SimplnxCore][VerifyTriangleWindingFilter]") { + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + const nx::core::UnitTest::TestFileSentinel testDataSentinel(nx::core::unit_test::k_TestFilesDir, "STL_Models.tar.gz", "STL_Models"); DataStructure dataStructure = {}; @@ -215,13 +315,37 @@ TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: Valid Region Ids Execution" args.insertOrAssign(VerifyTriangleWindingFilter::k_RepairNormals_Key, std::make_any(true)); args.insertOrAssign(VerifyTriangleWindingFilter::k_TriangleNormalsPath_Key, std::make_any(k_FaceDataPath.createChildPath(Constants::k_FaceNormals))); + const DataPath labelsPath = k_FaceDataPath.createChildPath(Constants::k_FeatureIds); + const DataPath normalsPath = k_FaceDataPath.createChildPath(Constants::k_FaceNormals); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_TriangleGeomPath)); + const auto& triangleGeom = dataStructure.getDataRefAs(k_TriangleGeomPath); + REQUIRE(triangleGeom.getVertices() != nullptr); + REQUIRE(triangleGeom.getFaces() != nullptr); + scope.requireExpectedStore(*triangleGeom.getVertices()); + scope.requireExpectedStore(*triangleGeom.getFaces()); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(labelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(normalsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(labelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(normalsPath)); + // Preflight the filter and check result auto preflightResult = filter.preflight(dataStructure, args); REQUIRE(preflightResult.outputActions.valid()); // Execute the filter and check the result - auto executeResult = filter.execute(dataStructure, args); + auto executeResult = scope.executeFilter(filter, dataStructure, args); REQUIRE(executeResult.result.valid()); + + REQUIRE_NOTHROW(dataStructure.getDataRefAs(k_TriangleGeomPath)); + const auto& repairedTriangleGeom = dataStructure.getDataRefAs(k_TriangleGeomPath); + REQUIRE(repairedTriangleGeom.getVertices() != nullptr); + REQUIRE(repairedTriangleGeom.getFaces() != nullptr); + scope.requireExpectedStore(*repairedTriangleGeom.getVertices()); + scope.requireExpectedStore(*repairedTriangleGeom.getFaces()); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(labelsPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs(normalsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(labelsPath)); + scope.requireExpectedStore(dataStructure.getDataRefAs(normalsPath)); } DataStructure exemplarDataStructure = {}; @@ -290,7 +414,7 @@ TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: Duplicate Vertices Catch", constexpr usize currentVert = 3; constexpr usize prevVert = 0; - // replace vert with a duplicate point (doesn't matter that it invalidates faces) + // Replace the vertex with a duplicate point. This test can invalidate faces. vertexList[currentVert] = vertexList[prevVert]; vertexList[currentVert + 1] = vertexList[prevVert + 1]; vertexList[currentVert + 2] = vertexList[prevVert + 2]; @@ -356,3 +480,341 @@ TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: SIMPL Backwards Compatibili } } } + +TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: winding repair preserves disabled normals", "[SimplnxCore][VerifyTriangleWindingFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + const bool useFaceLabels = GENERATE(false, true); + const bool hasNormals = GENERATE(false, true); + DYNAMIC_SECTION("Face labels: " << useFaceLabels << ", existing normals: " << hasNormals) + { + const DataPath geometryPath({"DisabledNormalsTetrahedron"}); + const DataPath faceDataPath = geometryPath.createChildPath("FaceData"); + const DataPath vertexDataPath = geometryPath.createChildPath("VertexData"); + const DataPath facesPath = geometryPath.createChildPath("Faces"); + const DataPath verticesPath = geometryPath.createChildPath("Vertices"); + const DataPath idsPath = faceDataPath.createChildPath("Ids"); + const DataPath normalsPath = faceDataPath.createChildPath("Normals"); + const usize idComponentCount = useFaceLabels ? 2 : 1; + const std::vector expectedIds = useFaceLabels ? std::vector{0, 1, 0, 1, 0, 1, 0, 1} : std::vector{1, 1, 1, 1}; + constexpr std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1}; + // Each expected edge has an opposite edge on its neighboring face. These faces enclose a unit tetrahedron with outward normals. + constexpr std::array expectedFaces = {0, 2, 1, 0, 1, 3, 0, 3, 2, 1, 2, 3}; + // Faces 1 and 3 start reversed; face 0 retains the outward seed orientation. + constexpr std::array inputFaces = {0, 2, 1, 3, 1, 0, 0, 3, 2, 3, 2, 1}; + constexpr std::array sentinelNormals = {11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43}; + REQUIRE(inputFaces != expectedFaces); + + DataStructure dataStructure; + auto* triangleGeomPtr = TriangleGeom::Create(dataStructure, geometryPath.getTargetName()); + REQUIRE(triangleGeomPtr != nullptr); + const auto geometryId = triangleGeomPtr->getId(); + auto* faceDataPtr = AttributeMatrix::Create(dataStructure, faceDataPath.getTargetName(), {4}, geometryId); + auto* vertexDataPtr = AttributeMatrix::Create(dataStructure, vertexDataPath.getTargetName(), {4}, geometryId); + REQUIRE(faceDataPtr != nullptr); + REQUIRE(vertexDataPtr != nullptr); + triangleGeomPtr->setFaceAttributeMatrix(*faceDataPtr); + triangleGeomPtr->setVertexAttributeMatrix(*vertexDataPtr); + + auto* verticesPtr = CreateDisabledNormalsArray(dataStructure, verticesPath, 3, nonstd::span(vertices.data(), vertices.size())); + auto* facesPtr = CreateDisabledNormalsArray(dataStructure, facesPath, 3, nonstd::span(inputFaces.data(), inputFaces.size())); + auto* idsPtr = CreateDisabledNormalsArray(dataStructure, idsPath, idComponentCount, nonstd::span(expectedIds.data(), expectedIds.size())); + triangleGeomPtr->setVertices(*verticesPtr); + triangleGeomPtr->setFaceList(*facesPtr); + const auto verticesId = verticesPtr->getId(); + const auto facesId = facesPtr->getId(); + const auto idsId = idsPtr->getId(); + const auto* verticesStorePtr = verticesPtr->getDataStore(); + const auto* facesStorePtr = facesPtr->getDataStore(); + const auto* idsStorePtr = idsPtr->getDataStore(); + std::optional normalsId; + const AbstractDataStore* normalsStorePtr = nullptr; + if(hasNormals) + { + auto* normalsPtr = CreateDisabledNormalsArray(dataStructure, normalsPath, 3, nonstd::span(sentinelNormals.data(), sentinelNormals.size())); + normalsId = normalsPtr->getId(); + normalsStorePtr = normalsPtr->getDataStore(); + } + + const auto checkUnchangedInputs = [&]() { + REQUIRE_NOTHROW(dataStructure.getDataRefAs(geometryPath)); + const auto& geometry = dataStructure.getDataRefAs(geometryPath); + REQUIRE(geometry.getId() == geometryId); + REQUIRE(geometry.getVertices() == verticesPtr); + REQUIRE(geometry.getFaces() == facesPtr); + REQUIRE(geometry.getFaceAttributeMatrix() == faceDataPtr); + REQUIRE(geometry.getVertexAttributeMatrix() == vertexDataPtr); + REQUIRE(dataStructure.getData(faceDataPath) == faceDataPtr); + REQUIRE(dataStructure.getData(vertexDataPath) == vertexDataPtr); + REQUIRE(geometry.getFaceAttributeMatrix()->getShape() == ShapeType{4}); + REQUIRE(geometry.getVertexAttributeMatrix()->getShape() == ShapeType{4}); + CheckDisabledNormalsArray(dataStructure, verticesPath, verticesId, verticesStorePtr, 3, nonstd::span(vertices.data(), vertices.size())); + CheckDisabledNormalsArray(dataStructure, idsPath, idsId, idsStorePtr, idComponentCount, nonstd::span(expectedIds.data(), expectedIds.size())); + if(hasNormals) + { + REQUIRE(normalsId.has_value()); + CheckDisabledNormalsArray(dataStructure, normalsPath, normalsId.value(), normalsStorePtr, 3, nonstd::span(sentinelNormals.data(), sentinelNormals.size())); + } + else + { + REQUIRE(dataStructure.getData(normalsPath) == nullptr); + } + }; + checkUnchangedInputs(); + CheckDisabledNormalsArray(dataStructure, facesPath, facesId, facesStorePtr, 3, nonstd::span(inputFaces.data(), inputFaces.size())); + + VerifyTriangleWindingFilter filter; + Arguments args; + args.insertOrAssign(VerifyTriangleWindingFilter::k_LabelsPath_Key, std::make_any(idsPath)); + args.insertOrAssign(VerifyTriangleWindingFilter::k_RepairNormals_Key, std::make_any(false)); + args.insertOrAssign(VerifyTriangleWindingFilter::k_TriangleNormalsPath_Key, std::make_any(normalsPath)); + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + scope.requireExpectedStore(*verticesPtr); + scope.requireExpectedStore(*facesPtr); + scope.requireExpectedStore(*idsPtr); + auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + checkUnchangedInputs(); + CheckDisabledNormalsArray(dataStructure, facesPath, facesId, facesStorePtr, 3, nonstd::span(expectedFaces.data(), expectedFaces.size())); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: resident late phase equivalence", "[SimplnxCore][VerifyTriangleWindingFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const usize idCase = GENERATE(usize{0}, usize{1}, usize{2}, usize{3}); + const bool reverseInput = GENERATE(false, true); + const bool repairNormals = GENERATE(false, true); + CAPTURE(scenario, idCase, reverseInput, repairNormals); + UnitTest::AlgorithmTestScope scope(scenario); + + const DataPath geometryPath({"LatePhaseTetrahedron"}); + const DataPath facesPath = geometryPath.createChildPath("Faces"); + const DataPath verticesPath = geometryPath.createChildPath("Vertices"); + const DataPath faceDataPath = geometryPath.createChildPath("FaceData"); + const DataPath idsPath = faceDataPath.createChildPath("Ids"); + const DataPath normalsPath = faceDataPath.createChildPath("Normals"); + constexpr std::array vertices = {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1}; + constexpr std::array outwardFaces = {0, 2, 1, 0, 1, 3, 0, 3, 2, 1, 2, 3}; + constexpr std::array sentinelNormals = {11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43}; + auto inputFaces = outwardFaces; + auto expectedFaces = outwardFaces; + for(usize face = 0; face < 4; ++face) + { + if(reverseInput) + { + std::swap(inputFaces[face * 3], inputFaces[face * 3 + 2]); + } + // A positive label in slot zero uses the opposite volume convention. + if(idCase == 3) + { + std::swap(expectedFaces[face * 3], expectedFaces[face * 3 + 2]); + } + } + const usize idComponents = idCase == 0 ? 1 : 2; + std::vector ids; + for(usize face = 0; face < 4; ++face) + { + if(idCase == 0) + { + ids.push_back(3); + } + else if(idCase == 1) + { + ids.insert(ids.end(), {0, 3}); + } + else if(idCase == 2) + { + ids.insert(ids.end(), {-1, 3}); + } + else + { + ids.insert(ids.end(), {3, -1}); + } + } + + DataStructure dataStructure; + auto* geometry = TriangleGeom::Create(dataStructure, geometryPath.getTargetName()); + REQUIRE(geometry != nullptr); + auto* faceData = AttributeMatrix::Create(dataStructure, faceDataPath.getTargetName(), {4}, geometry->getId()); + REQUIRE(faceData != nullptr); + geometry->setFaceAttributeMatrix(*faceData); + auto* vertexArray = CreateDisabledNormalsArray(dataStructure, verticesPath, 3, {vertices.data(), vertices.size()}); + auto* faceArray = CreateDisabledNormalsArray(dataStructure, facesPath, 3, {inputFaces.data(), inputFaces.size()}); + auto* idsArray = CreateDisabledNormalsArray(dataStructure, idsPath, idComponents, {ids.data(), ids.size()}); + auto* normalsArray = CreateDisabledNormalsArray(dataStructure, normalsPath, 3, {sentinelNormals.data(), sentinelNormals.size()}); + geometry->setVertices(*vertexArray); + geometry->setFaceList(*faceArray); + const auto vertexId = vertexArray->getId(); + const auto faceId = faceArray->getId(); + const auto idsId = idsArray->getId(); + const auto normalsId = normalsArray->getId(); + const auto* vertexStore = vertexArray->getDataStore(); + const auto* faceStore = faceArray->getDataStore(); + const auto* idsStore = idsArray->getDataStore(); + const auto* normalsStore = normalsArray->getDataStore(); + scope.requireExpectedStore(*vertexArray); + scope.requireExpectedStore(*faceArray); + scope.requireExpectedStore(*idsArray); + scope.requireExpectedStore(*normalsArray); + + VerifyTriangleWindingFilter filter; + Arguments args; + args.insertOrAssign(VerifyTriangleWindingFilter::k_LabelsPath_Key, std::make_any(idsPath)); + args.insertOrAssign(VerifyTriangleWindingFilter::k_RepairNormals_Key, std::make_any(repairNormals)); + args.insertOrAssign(VerifyTriangleWindingFilter::k_TriangleNormalsPath_Key, std::make_any(normalsPath)); + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + CheckDisabledNormalsArray(dataStructure, verticesPath, vertexId, vertexStore, 3, {vertices.data(), vertices.size()}); + CheckDisabledNormalsArray(dataStructure, facesPath, faceId, faceStore, 3, {expectedFaces.data(), expectedFaces.size()}); + CheckDisabledNormalsArray(dataStructure, idsPath, idsId, idsStore, idComponents, {ids.data(), ids.size()}); + if(repairNormals) + { + REQUIRE_NOTHROW(dataStructure.getDataRefAs(normalsPath)); + const auto& normals = dataStructure.getDataRefAs(normalsPath); + REQUIRE(normals.getId() == normalsId); + REQUIRE(normals.getDataStore() == normalsStore); + REQUIRE(normals.getTupleShape() == ShapeType{4}); + REQUIRE(normals.getComponentShape() == ShapeType{3}); + const float64 diagonal = 1.0 / std::sqrt(3.0); + const std::array outwardNormals = {0, 0, -1, 0, -1, 0, -1, 0, 0, diagonal, diagonal, diagonal}; + const float64 sign = idCase == 3 ? -1.0 : 1.0; + for(usize index = 0; index < outwardNormals.size(); ++index) + { + if(index < 9) + { + CHECK(normals[index] == sign * outwardNormals[index]); + } + else + { + // Both paths normalize in float64; this tolerates only final rounding. + CHECK(normals[index] == Approx(sign * outwardNormals[index]).margin(1.0e-14)); + } + } + } + else + { + CheckDisabledNormalsArray(dataStructure, normalsPath, normalsId, normalsStore, 3, {sentinelNormals.data(), sentinelNormals.size()}); + } + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("SimplnxCore::VerifyTriangleWindingFilter: float32 volume accumulation order", "[SimplnxCore][VerifyTriangleWindingFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool reordered = GENERATE(false, true); + const usize blockCapacity = GENERATE(usize{1}, usize{3}); + CAPTURE(scenario, reordered, blockCapacity); + UnitTest::AlgorithmTestScope scope(scenario); + + const DataPath geometryPath({"VolumeOrderTriangles"}); + const DataPath facesPath = geometryPath.createChildPath("Faces"); + const DataPath verticesPath = geometryPath.createChildPath("Vertices"); + const DataPath faceDataPath = geometryPath.createChildPath("FaceData"); + const DataPath idsPath = faceDataPath.createChildPath("Ids"); + const DataPath normalsPath = faceDataPath.createChildPath("UnusedNormals"); + constexpr std::array vertices = {0, 0, 1, 6, 0, 1, 0, 16777216, 1, 16, 0, 1, 22, 0, 1, 16, 1, 1, 32, 0, 1, 38, 0, 1, 32, 16777216, 1}; + std::array inputFaces = {0, 2, 1, 3, 5, 4, 6, 7, 8}; + constexpr std::array ids = {1, 1, 1}; + if(reordered) + { + for(usize corner = 0; corner < 3; ++corner) + { + std::swap(inputFaces[3 + corner], inputFaces[6 + corner]); + } + } + auto expectedFaces = inputFaces; + // Disconnected triangles have no shared edges for the initial repair to change. + // Original region contributions are {-2^24, -1, +2^24}: float32 left additions give zero. + // In row order {first, third, second}, the sum is -1, so every face must reverse. + // A double accumulator or regrouped additions incorrectly reverses the original order. + if(reordered) + { + for(usize face = 0; face < 3; ++face) + { + std::swap(expectedFaces[face * 3], expectedFaces[face * 3 + 2]); + } + } + + DataStructure dataStructure; + auto* geometry = TriangleGeom::Create(dataStructure, geometryPath.getTargetName()); + REQUIRE(geometry != nullptr); + auto* faceData = AttributeMatrix::Create(dataStructure, faceDataPath.getTargetName(), {3}, geometry->getId()); + REQUIRE(faceData != nullptr); + geometry->setFaceAttributeMatrix(*faceData); + auto vertexStore = std::make_shared>(ShapeType{9}, ShapeType{3}, 0.0f); + auto faceStore = std::make_shared>(ShapeType{3}, ShapeType{3}, IGeometry::MeshIndexType{0}); + auto idsStore = std::make_shared>(ShapeType{3}, ShapeType{1}, int32{0}); + auto vertexWrite = vertexStore->copyFromBuffer(0, {vertices.data(), vertices.size()}); + auto faceWrite = faceStore->copyFromBuffer(0, {inputFaces.data(), inputFaces.size()}); + auto idsWrite = idsStore->copyFromBuffer(0, {ids.data(), ids.size()}); + SIMPLNX_RESULT_REQUIRE_VALID(vertexWrite); + SIMPLNX_RESULT_REQUIRE_VALID(faceWrite); + SIMPLNX_RESULT_REQUIRE_VALID(idsWrite); + auto* vertexArray = Float32Array::Create(dataStructure, verticesPath.getTargetName(), vertexStore, geometry->getId()); + auto* faceArray = TriangleGeom::SharedFaceList::Create(dataStructure, facesPath.getTargetName(), faceStore, geometry->getId()); + auto* idsArray = Int32Array::Create(dataStructure, idsPath.getTargetName(), idsStore, faceData->getId()); + REQUIRE(vertexArray != nullptr); + REQUIRE(faceArray != nullptr); + REQUIRE(idsArray != nullptr); + geometry->setVertices(*vertexArray); + geometry->setFaceList(*faceArray); + scope.requireExpectedStore(*vertexArray); + scope.requireExpectedStore(*faceArray); + scope.requireExpectedStore(*idsArray); + + VerifyTriangleWindingFilter filter; + Arguments args; + args.insertOrAssign(VerifyTriangleWindingFilter::k_LabelsPath_Key, std::make_any(idsPath)); + args.insertOrAssign(VerifyTriangleWindingFilter::k_RepairNormals_Key, std::make_any(false)); + args.insertOrAssign(VerifyTriangleWindingFilter::k_TriangleNormalsPath_Key, std::make_any(normalsPath)); + auto preflightResult = filter.preflight(dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(preflightResult.outputActions); + + auto& budget = CacheMemoryBudgetManager::instance(); + const uint64 reservedBefore = budget.reservedWorkingMemoryBytes(); + { + CacheMemoryBudgetManager::WorkingMemoryReservation occupied; + const uint64 bytesPerTriangle = 3 * sizeof(IGeometry::MeshIndexType) + 3 * sizeof(usize) + 18 * sizeof(float32) + sizeof(int32); + const uint64 targetGrant = blockCapacity * bytesPerTriangle; + if(scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore) + { + REQUIRE(budget.maximumWorkingMemoryBytes() >= reservedBefore + targetGrant); + const uint64 occupyBytes = budget.maximumWorkingMemoryBytes() - reservedBefore - targetGrant; + occupied = budget.reserveWorkingMemory(occupyBytes); + REQUIRE(occupied.sizeBytes() == occupyBytes); + } + uint64 observedGrant = 0; + const IFilter::MessageHandler messageHandler{[&](const IFilter::Message& message) { + if(message.message == "Scanning feature IDs") + { + observedGrant = budget.reservedWorkingMemoryBytes() - reservedBefore - occupied.sizeBytes(); + } + }}; + auto executeResult = scope.executeFilter(filter, dataStructure, args, nullptr, messageHandler); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + if(scenario == UnitTest::AlgorithmTestScenario::OutOfCoreAlgorithmOnInMemoryStore) + { + CHECK(observedGrant == targetGrant); + } + CHECK(budget.reservedWorkingMemoryBytes() == reservedBefore + occupied.sizeBytes()); + } + CHECK(budget.reservedWorkingMemoryBytes() == reservedBefore); + std::array actualFaces = {}; + auto faceRead = faceArray->getDataStoreRef().copyIntoBuffer(0, {actualFaces.data(), actualFaces.size()}); + SIMPLNX_RESULT_REQUIRE_VALID(faceRead); + CHECK(actualFaces == expectedFaces); + CHECK(dataStructure.getData(normalsPath) == nullptr); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} diff --git a/src/Plugins/SimplnxCore/test/WriteAvizoRectilinearCoordinateTest.cpp b/src/Plugins/SimplnxCore/test/WriteAvizoRectilinearCoordinateTest.cpp index 24ca5c2d69..2ea307dd5d 100644 --- a/src/Plugins/SimplnxCore/test/WriteAvizoRectilinearCoordinateTest.cpp +++ b/src/Plugins/SimplnxCore/test/WriteAvizoRectilinearCoordinateTest.cpp @@ -1,3 +1,4 @@ +#include "AvizoWriterFailureTestUtilities.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" #include @@ -24,6 +25,8 @@ using namespace nx::core::Constants; TEST_CASE("SimplnxCore::WriteAvizoRectilinearCoordinateFilter: Valid Filter Execution", "[SimplnxCore][WriteAvizoRectilinearCoordinateFilter]") { + UnitTest::LoadPlugins(); + const std::string kDataInputArchive = "6_6_avizo_writers.tar.gz"; const std::string kExpectedOutputTopLevel = "6_6_avizo_writers"; const nx::core::UnitTest::TestFileSentinel testDataSentinel1(nx::core::unit_test::k_TestFilesDir, kDataInputArchive, kExpectedOutputTopLevel); @@ -165,3 +168,16 @@ TEST_CASE("SimplnxCore::WriteAvizoRectilinearCoordinateFilter: Geometry cell tup REQUIRE(errorRef.message.find("'Image' has 12 cells") != std::string::npos); } } + +TEST_CASE("SimplnxCore::WriteAvizoRectilinearCoordinateFilter: Source read failure preserves destination", "[SimplnxCore][WriteAvizoRectilinearCoordinateFilter][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool writeBinary = GENERATE(false, true); + const bool existingDestination = GENERATE(false, true); + const bool throwAllocation = GENERATE(false, true); + DYNAMIC_SECTION("binary=" << writeBinary << ", existing destination=" << existingDestination << ", allocation failure=" << throwAllocation) + { + auto store = std::make_shared>>(ShapeType{1, 1, 65537}, ShapeType{1}, std::optional{7}); + UnitTest::AvizoWriterFailure::CheckSecondBlockFailure(writeBinary, existingDestination, store, throwAllocation); + } +} diff --git a/src/Plugins/SimplnxCore/test/WriteAvizoUniformCoordinateTest.cpp b/src/Plugins/SimplnxCore/test/WriteAvizoUniformCoordinateTest.cpp index d2bed71c6f..459985054a 100644 --- a/src/Plugins/SimplnxCore/test/WriteAvizoUniformCoordinateTest.cpp +++ b/src/Plugins/SimplnxCore/test/WriteAvizoUniformCoordinateTest.cpp @@ -1,3 +1,4 @@ +#include "AvizoWriterFailureTestUtilities.hpp" #include "SimplnxCore/SimplnxCore_test_dirs.hpp" #include @@ -166,3 +167,16 @@ TEST_CASE("SimplnxCore::WriteAvizoUniformCoordinateFilter: Geometry cell tuple c REQUIRE(errorRef.message.find("'Image' has 12 cells") != std::string::npos); } } + +TEST_CASE("SimplnxCore::WriteAvizoUniformCoordinateFilter: Source read failure preserves destination", "[SimplnxCore][WriteAvizoUniformCoordinateFilter][AvizoWriter]") +{ + UnitTest::LoadPlugins(); + const bool writeBinary = GENERATE(false, true); + const bool existingDestination = GENERATE(false, true); + const bool throwAllocation = GENERATE(false, true); + DYNAMIC_SECTION("binary=" << writeBinary << ", existing destination=" << existingDestination << ", allocation failure=" << throwAllocation) + { + auto store = std::make_shared>>(ShapeType{1, 1, 65537}, ShapeType{1}, std::optional{7}); + UnitTest::AvizoWriterFailure::CheckSecondBlockFailure(writeBinary, existingDestination, store, throwAllocation); + } +} diff --git a/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTest.cpp b/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTest.cpp index 57575c94e0..f78261fe86 100644 --- a/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTest.cpp +++ b/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTest.cpp @@ -1,3 +1,5 @@ +#include "WriteVtkStructuredPointsTestUtilities.hpp" + #include #include "SimplnxCore/Filters/WriteVtkStructuredPointsFilter.hpp" @@ -198,3 +200,12 @@ TEST_CASE("SimplnxCore::WriteVtkStructuredPointsFilter: cancellation leaves the const std::string contents((std::istreambuf_iterator(inputStream)), std::istreambuf_iterator()); REQUIRE(contents == sentinel); } + +TEST_CASE("SimplnxCore::WriteVtkStructuredPointsFilter: ASCII correctness", "[SimplnxCore][WriteVtkStructuredPointsFilter]") +{ + UnitTest::LoadPlugins(); + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + CAPTURE(scenario); + UnitTest::AlgorithmTestScope scope(scenario); + WriteVtkStructuredPointsTest::CheckAsciiOutput(scope, ""); +} diff --git a/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTestUtilities.hpp b/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTestUtilities.hpp new file mode 100644 index 0000000000..343cf754fe --- /dev/null +++ b/src/Plugins/SimplnxCore/test/WriteVtkStructuredPointsTestUtilities.hpp @@ -0,0 +1,165 @@ +#pragma once + +#include "SimplnxCore/Filters/WriteVtkStructuredPointsFilter.hpp" +#include "SimplnxCore/SimplnxCore_test_dirs.hpp" + +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Uuid.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/Geometry/ImageGeom.hpp" +#include "simplnx/Parameters/FileSystemPathParameter.hpp" +#include "simplnx/Parameters/MultiArraySelectionParameter.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/** + * @namespace WriteVtkStructuredPointsTest + * @brief Checks the same ASCII file contract for resident and disk-backed inputs. + */ +namespace WriteVtkStructuredPointsTest +{ +/** + * @brief Writes two arrays and checks every output field and unchanged input value. + * @tparam ScopeT Provides the selected algorithm and storage checks. + * @param scope Owns the storage policy before this function creates arrays. + * @param expectedFormat Identifies the required backing format for both input arrays. + */ +template +void CheckAsciiOutput(ScopeT& scope, const std::string& expectedFormat) +{ + using namespace nx::core; + namespace fs = std::filesystem; + + constexpr std::array k_ExpectedSignedValues = {-128, 127, -7, 9, 0, 42, 100, -100}; + constexpr std::array k_ExpectedFloatValues = {0.25F, -1.5F, 2.0F, 100.0F}; + const DataPath geomPath({"ASCII ImageGeom"}); + const DataPath cellDataPath = geomPath.createChildPath("Cell Data"); + const DataPath signedPath = cellDataPath.createChildPath("Signed Components"); + const DataPath floatPath = cellDataPath.createChildPath("Float Scalars"); + const ShapeType tupleShape = {1, 2, 2}; + + DataStructure dataStructure; + auto* imageGeom = ImageGeom::Create(dataStructure, "ASCII ImageGeom"); + REQUIRE(imageGeom != nullptr); + imageGeom->setDimensions({2, 2, 1}); + imageGeom->setOrigin({-1.0F, 2.0F, 3.0F}); + imageGeom->setSpacing({0.5F, 1.5F, 2.0F}); + auto* cellData = AttributeMatrix::Create(dataStructure, "Cell Data", tupleShape, imageGeom->getId()); + REQUIRE(cellData != nullptr); + imageGeom->setCellData(*cellData); + + auto signedStore = DataStoreUtilities::CreateDataStore(dataStructure, signedPath, tupleShape, {2}); + REQUIRE(signedStore != nullptr); + auto* signedArray = Int8Array::Create(dataStructure, "Signed Components", signedStore, cellData->getId()); + REQUIRE(signedArray != nullptr); + const auto signedWrite = signedStore->copyFromBuffer(0, nonstd::span(k_ExpectedSignedValues.data(), k_ExpectedSignedValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(signedWrite); + + auto floatStore = DataStoreUtilities::CreateDataStore(dataStructure, floatPath, tupleShape, {1}); + REQUIRE(floatStore != nullptr); + auto* floatArray = Float32Array::Create(dataStructure, "Float Scalars", floatStore, cellData->getId()); + REQUIRE(floatArray != nullptr); + const auto floatWrite = floatStore->copyFromBuffer(0, nonstd::span(k_ExpectedFloatValues.data(), k_ExpectedFloatValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(floatWrite); + + scope.requireExpectedStore(*signedArray); + scope.requireExpectedStore(*floatArray); + REQUIRE(signedArray->getDataFormat() == expectedFormat); + REQUIRE(floatArray->getDataFormat() == expectedFormat); + + const fs::path outputBase = fs::path(unit_test::k_BinaryTestOutputDir.view()); + fs::create_directories(outputBase); + const fs::path outputRoot = outputBase / ("WriteVtkAscii-" + Uuid::GenerateV4().str()); + REQUIRE(fs::create_directory(outputRoot)); + auto outputGuard = MakeScopeGuard([&outputRoot]() noexcept { + std::error_code errorCode; + fs::remove_all(outputRoot, errorCode); + }); + const fs::path outputPath = outputRoot / "output.vtk"; + + WriteVtkStructuredPointsFilter filter; + Arguments args = filter.getDefaultArguments(); + args.insertOrAssign(WriteVtkStructuredPointsFilter::k_OutputFile_Key, std::make_any(outputPath)); + args.insertOrAssign(WriteVtkStructuredPointsFilter::k_WriteBinaryFile_Key, std::make_any(false)); + args.insertOrAssign(WriteVtkStructuredPointsFilter::k_ImageGeometryPath_Key, std::make_any(geomPath)); + args.insertOrAssign(WriteVtkStructuredPointsFilter::k_SelectedDataArrayPaths_Key, + std::make_any(MultiArraySelectionParameter::ValueType{signedPath, floatPath})); + const auto executeResult = scope.executeFilter(filter, dataStructure, args); + SIMPLNX_RESULT_REQUIRE_VALID(executeResult.result); + + { + std::ifstream vtkFile(outputPath, std::ios::binary); + REQUIRE(vtkFile.is_open()); + vtkFile.imbue(std::locale::classic()); + constexpr std::array k_ExpectedHeaderLines = {"# vtk DataFile Version 3.0", "vtk output", "ASCII", "DATASET STRUCTURED_POINTS", "DIMENSIONS 3 3 2", + "SPACING 0.5 1.5 2", "ORIGIN -1 2 3", "CELL_DATA 4", "SCALARS Signed_Components char 2", "LOOKUP_TABLE default"}; + std::string line; + for(const char* expectedLine : k_ExpectedHeaderLines) + { + REQUIRE(std::getline(vtkFile, line)); + REQUIRE(line == expectedLine); + } + + // Integer extraction must reject character output for signed 8-bit values. + for(const int8 expectedValue : k_ExpectedSignedValues) + { + int actualValue = 0; + REQUIRE(vtkFile >> actualValue); + REQUIRE(actualValue == static_cast(expectedValue)); + } + vtkFile >> std::ws; + REQUIRE(std::getline(vtkFile, line)); + REQUIRE(line == "SCALARS Float_Scalars float 1"); + REQUIRE(std::getline(vtkFile, line)); + REQUIRE(line == "LOOKUP_TABLE default"); + for(const float32 expectedValue : k_ExpectedFloatValues) + { + double actualValue = 0.0; + REQUIRE(vtkFile >> actualValue); + REQUIRE(actualValue == static_cast(expectedValue)); + } + vtkFile >> std::ws; + REQUIRE(vtkFile.peek() == std::ifstream::traits_type::eof()); + REQUIRE_FALSE(vtkFile.bad()); + } + + scope.requireExpectedStore(*signedArray); + scope.requireExpectedStore(*floatArray); + REQUIRE(signedArray->getDataFormat() == expectedFormat); + REQUIRE(floatArray->getDataFormat() == expectedFormat); + REQUIRE(signedArray->getDataType() == DataType::int8); + REQUIRE(floatArray->getDataType() == DataType::float32); + REQUIRE(signedArray->getTupleShape() == tupleShape); + REQUIRE(floatArray->getTupleShape() == tupleShape); + REQUIRE(signedArray->getComponentShape() == ShapeType{2}); + REQUIRE(floatArray->getComponentShape() == ShapeType{1}); + std::array actualSignedValues = {}; + const auto signedRead = signedArray->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualSignedValues.data(), actualSignedValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(signedRead); + REQUIRE(actualSignedValues == k_ExpectedSignedValues); + std::array actualFloatValues = {}; + const auto floatRead = floatArray->getDataStoreRef().copyIntoBuffer(0, nonstd::span(actualFloatValues.data(), actualFloatValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(floatRead); + REQUIRE(actualFloatValues == k_ExpectedFloatValues); + + const SizeVec3 expectedDims = {2, 2, 1}; + const FloatVec3 expectedOrigin = {-1.0F, 2.0F, 3.0F}; + const FloatVec3 expectedSpacing = {0.5F, 1.5F, 2.0F}; + REQUIRE(imageGeom->getDimensions() == expectedDims); + REQUIRE(imageGeom->getOrigin() == expectedOrigin); + REQUIRE(imageGeom->getSpacing() == expectedSpacing); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} +} // namespace WriteVtkStructuredPointsTest diff --git a/src/Plugins/SimplnxCore/vv/ComputeFeatureCentroidsFilter.md b/src/Plugins/SimplnxCore/vv/ComputeFeatureCentroidsFilter.md index bf0e7881cf..95a9dabe5e 100644 --- a/src/Plugins/SimplnxCore/vv/ComputeFeatureCentroidsFilter.md +++ b/src/Plugins/SimplnxCore/vv/ComputeFeatureCentroidsFilter.md @@ -15,16 +15,18 @@ | Aspect | Current state | |------------------------|--------------------------| | Algorithm Relationship | **Port** — the non-periodic path of `ComputeFeatureCentroids::operator()` is a near line-by-line translation of legacy `FindFeatureCentroids::find_centroids()`, including the identical Kahan compensated summation and `float64`/`float64`/`uint64` sum/compensation/count triplet. Port-time additions: SIMPLNX-only `Is Periodic` option, disabled `ParallelDataAlgorithm` scaffolding, `float64` voxel-center fetch (`Point3Dd`), and a FeatureIds→AttributeMatrix indexing guard. | -| Oracle (confirmed) | **Class 1 (Analytical) primary + Class 4 (Invariant) companion** (confirmed 2026-07-07). 6 hand-derived toy fixtures A–F cover arithmetic centroids, minimum-image periodic centroids, and the domain-filling fallback. Signed off by Michael Jackson (technical authority) 2026-07-16. | +| Oracle (confirmed) | **Class 1 (Analytical) primary + Class 4 (Invariant) companion** (confirmed 2026-07-07). Historical fixtures A–F cover arithmetic centroids and several periodic cases. The 2026-10 correction adds discriminating unequal-population, tied-gap, full-axis, and sorted-coordinate oracles. The historical 2026-07-16 sign-off is retained. | | Code paths enumerated | 9 of 9 listed numerical/validation paths exercised; cancellation and internal error-site exhaustiveness are outside this campaign. | -| Tests today | 5 registered cases plus 1 hidden real-HDF5 block-tail case, which runs periodic mode off and on. Serial CTest passes 5/5 in both builds. | +| Tests today | The 2026-10-15 Windows run passed all 8 registered cases in in-core Release and OOC Release/Debug. Both hidden HDF5 cases also passed, normally and with optional working-memory probes. The historical 2026-09-21 results remain recorded below. | | Exemplar archive | **None — inline analytical fixtures** (provenance sidecar written). The shared `6_6_stats_test_v2.tar.gz` is no longer consumed by this test (kept for 5 other tests); `6_6_find_feature_centroids.tar.gz` kept (used by ExtractComponentAsArray / WriteAbaqusHexahedron). | | Legacy comparison | **Source-inspection** (DREAM3D 6.5.171 vs SIMPLNX; local legacy source used for corroboration), backed by the independent Class 1 fixtures. Non-periodic path is an exact Kahan port; 2 deviations (D1 float32→float64 voxel-center precision; D2 SIMPLNX-only `Is Periodic`). Empirical binary A/B available if bit-confirmation is required. | -| Bug flags | **None outstanding.** Two SIMPLNX-internal periodic-centroid defects were resolved: the original offset ignored spacing, and the constant half-domain offset model was incorrect for asymmetric wrapped features. The feature-centroid path now uses a minimum-image circular mean; the triangle-centroid utility uses a largest-empty-gap minimum-image mean. Fixtures D/E/F pin the corrected behavior. These are not legacy deviations because legacy has no periodic path. | +| Bug flags | The 2026-10 adversarial review found that the circular estimator introduced in 2026-07 disagreed with the documented unwrapped arithmetic mean for unequal voxel populations. The current periodic path uses the unique largest empty gap and a weighted arithmetic mean. The historical spacing and constant-offset fixes remain recorded below. | | V&V phase | Historical COMPLETE status and sign-off retained. OOC recertification verifies preserved analytical tests and a real-HDF5 block-tail case. | ## Follow-up (2026-07, issue #1665) — periodic model reimplemented +This section records what was implemented and verified in July 2026. Its circular-estimator description is superseded by the October 2026 correction below; the original verification and sign-off history is unchanged. + The original V&V (#1658) fixed a **spacing** sub-bug in the periodic path but validated the periodic result against the **constant half-domain offset model itself** (Fixture D's expected value was "naive + (dim−1)/2", a circular oracle — the test was designed around the code). An adversarial review (#1665) showed that model is fundamentally wrong: a constant offset is correct only when a wrapped feature's mass is symmetric about the seam, and for asymmetric features it places the centroid in the empty middle of the feature or outside the domain entirely. This follow-up (on top of merged #1658) corrects it: @@ -36,13 +38,25 @@ This follow-up (on top of merged #1658) corrects it: The non-periodic path is unchanged (still the verified Kahan port). Deviation `ComputeFeatureCentroidsFilter-D2` is updated accordingly. +## Follow-up (2026-10) — exact periodic arithmetic + +An adversarial five-cell fixture in a 5×3×1 image exposed a difference between the circular estimator (4.75) and the documented arithmetic mean after unwrapping (4.7). The feature's X populations are two cells at index 0, one at 3, and two at 4. The unique largest empty gap lies between indices 0 and 3; shifting index 0 by one period and averaging all five cells gives 4.7. A separate D=8 unequal-population fixture distinguishes this rule from recentering around a circular anchor. + +The current implementation records occupied coordinates for actual boundary-spanning feature-axis pairs in bounded bitsets. It carries largest-gap and tie state across ascending words and windows, then makes one sequential weighted Kahan pass for axes with a unique largest gap. Tied gaps and complete axis occupancy retain the original arithmetic mean. The original nonperiodic Kahan summation and unused-feature zero behavior remain. The new occupancy payload requests up to 32 MiB of shared working memory, adapts to the grant, and requires a minimum 8-byte word; validation and pre-final-write failures leave final centroid publication untouched. + +The registered tests include the five-cell result, axis permutations, signed/nonunit spacing and origin, all-axis wrapping, ties, full occupancy, deterministic sorted-coordinate random cases, exact small-grant/window behavior, and cancellation/read/resource failures. A separate hidden test checks the wrapped result with actual HDF5-OOC Feature IDs and centroid stores. These cases correct the current oracle claim without retroactively changing the July or September test record. + +Verification on 2026-10-15 used Windows MSVC 14.44.35207 with the paired DREAM3D-NX builds: in-core Release (`SIMPLNX_TEST_ALGORITHM_PATH=0`) and OOC Release/Debug (`SIMPLNX_TEST_ALGORITHM_PATH=1`). All eight registered centroid cases passed in each configuration, including the metadata-only arithmetic-overflow guards. The two hidden HDF5 cases passed in Release and Debug with 54 assertions normally and 396 assertions with the optional working-memory probes. Both input and output stores are checked as HDF5-OOC. These are local Windows results, not native macOS/Linux results or a new technical-authority sign-off. + +The optional probe uses a 32768-by-16 image and a fixed 256 MiB cache budget. Its occupancy needs only 65,544 bytes, so 8/32/64 MiB of available working memory all fit one batch. A 64 KiB grant requires two batches and an 8 KiB grant requires nine. The resident probe observes the corresponding logical-read counts; the HDF5 probe labels those batch counts as planned and checks every output component. Release samples were approximately 9.6-9.8 ms with ample resident working memory and 35.1 ms at 8 KiB; the HDF5 samples were 14.1-23.4 ms and 39.4 ms respectively. These are single, likely warm-cache samples taken while other builds could run. They demonstrate correct results with smaller grants, not cold-cache throughput, a comparison against the old estimator, large-pipeline performance, or an optimal 32 MiB limit. + ## Summary `ComputeFeatureCentroidsFilter` computes the centroid of each feature as the average X/Y/Z position of all cells belonging to that feature, using Kahan compensated summation for numerical precision. An optional -`Is Periodic` mode uses a minimum-image circular mean for features that span the image boundary. Verification -used a **Class 1 (Analytical)** oracle — arithmetic means for non-periodic features and independently derived -minimum-image means for wrapped features — paired with **Class 4 (Invariant)** bounding-box, fire-condition, +`Is Periodic` mode uses the unique largest-gap arithmetic mean for features that span the image boundary. Current verification +uses a **Class 1 (Analytical)** oracle — arithmetic means for non-periodic features and independently derived +sorted-coordinate weighted means for wrapped features — paired with **Class 4 (Invariant)** bounding-box, fire-condition, and in-domain predicates. SIMPLNX matched the analytical values on every fixture; the legacy diff reduces to a single float32-vs-float64 precision non-deviation plus the SIMPLNX-only periodic feature, and the reconciliation surfaced and fixed two SIMPLNX-internal periodic-centroid defects. @@ -60,8 +74,8 @@ triplet, and the `count>0` divide-and-store finalize. Same SIMPL UUID retained *Port-time deltas:* -1. **`Is Periodic` option** (SIMPLNX-only) + per-feature index-range tracking and unit-vector - accumulation for a minimum-image circular mean. Changes output when enabled; default `false` reproduces +1. **`Is Periodic` option** (SIMPLNX-only) + per-feature index-range tracking and bounded occupied-coordinate + scans for the unique largest-gap unwrapped arithmetic mean. Changes output when enabled; default `false` reproduces legacy exactly. Legacy has no equivalent. Drives deviation D2. 2. **Voxel-center precision**: SIMPLNX fetches the voxel center directly in `float64` (`ImageGeom::getCoords` → `Point3Dd`); legacy fetches into a `float[3]` then promotes to `double` inside the @@ -74,7 +88,9 @@ triplet, and the `count>0` divide-and-store finalize. Same SIMPL UUID retained *Material PRs since baseline (2025-10-01):* none change the algorithm. #1547 (doc: `Is Periodic` description), #1543 (doc: pipeline rename), #1588 (test: SIMPL 6.4/6.5 backwards-compat + 2 fixtures). -## Oracle +## Oracle recorded in the 2026-07 V&V + +The following records the oracle and conclusions as signed off in July. Its circular-mean interpretation and periodic adjustment predicate are superseded by the 2026-10 follow-up above. *Class:* **1 (Analytical)** primary + **4 (Invariant)** companion. (Confirmed by developer 2026-07-07.) Class 3 N/A — no published paper for "mean position of the cells in a feature". @@ -95,11 +111,11 @@ all pass at float32 margin 1e-4. *Second-engineer review:* **Signed off by Michael Jackson (technical authority), 2026-07-16.** -## Bugs found and fixed +## Bugs found and fixed in the historical V&V No new defect was found during this OOC recertification. The historical periodic-spacing correction described above remains in the branch. -## Code path coverage +## Code path coverage recorded in the historical V&V 9 of 9 listed numerical/validation paths exercised. Source: `src/Plugins/SimplnxCore/src/SimplnxCore/Filters/Algorithms/ComputeFeatureCentroids.cpp`. @@ -122,7 +138,7 @@ adjust. cannot be triggered with valid arguments — not separately tested. The OOC dual-build run is deferred: the algorithm is a feature-indexed serial sweep with no OOC-specific variant.* -## Test inventory +## Test inventory recorded in the historical V&V | Test case | Status | Notes | |-----------|--------|-------| @@ -134,7 +150,7 @@ algorithm is a feature-indexed serial sweep with no OOC-specific variant.* | `Bulk boundary and failure propagation` | kept | Existing bulk-read boundary and injected-failure regression. | | `real HDF5 65536-block tail oracle` | new-for-V&V | Reuses CentroidToy::Build and Run; checks exact physical centroids for background, the two-cell feature across the boundary, and an empty feature. | -All active TEST_CASEs pass in-core (`NX-Com-Qt69-Vtk96-Rel`). OOC dual-build deferred (see Code path coverage). +At the time of this historical inventory, all active TEST_CASEs passed in-core (`NX-Com-Qt69-Vtk96-Rel`). OOC dual-build was deferred then (see historical Code path coverage). OOC recertification (2026-09-21): 5/5 registered CTest entries pass in each DREAM3DNX build. The new hidden case passes 35 assertions across periodic off/on. Input FeatureIds and output Centroids are asserted HDF5-OOC. The two contributing X coordinates are 131081 and 131083, so the expected mean is exactly 131082. Upstream numerical assertions and tolerances remain unchanged; legacy deviation dispositions are retained without claiming a new binary comparison. diff --git a/src/Plugins/SimplnxCore/vv/MultiThresholdObjectsFilter.md b/src/Plugins/SimplnxCore/vv/MultiThresholdObjectsFilter.md index 1e6bd0c5e8..58833da8e9 100644 --- a/src/Plugins/SimplnxCore/vv/MultiThresholdObjectsFilter.md +++ b/src/Plugins/SimplnxCore/vv/MultiThresholdObjectsFilter.md @@ -14,9 +14,9 @@ | Aspect | Current state | |------------------------|--------------------------| | Algorithm Relationship | **Rewrite.** Consolidates two independently-shipped legacy filters — **Threshold Objects** (flat, AND-only) and **Threshold Objects (Advanced)** (nested AND/OR sets) — into one SIMPLNX filter under one new UUID, unified around a single `ArrayThresholdSet` model. Not a line-by-line translation of either legacy source. | -| Oracle (confirmed) | **Class 1 (Analytical) — confirmed.** `expected[i] = COMPARISON(input[i], value)`, hand-combined via AND/OR/invert boolean algebra. Encoded across 14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations in `MultiThresholdObjectsTest.cpp`, all pass. | +| Oracle (confirmed) | **Class 1 (Analytical) — confirmed.** `expected[i] = COMPARISON(input[i], value)`, hand-combined via AND/OR/invert boolean algebra. Encoded across 14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations in `MultiThresholdObjectsVvTest.cpp`; the 2026-08-19 sign-off records them as passing. | | Code paths enumerated | **26 of 27 exercised.** Row 13 (unreachable comparison-operator `else`-throw) is a permanent, acceptable gap and the only one remaining. Row 25 (a set mixing a leaf threshold with a nested set — the `MultiThresholdObjectsFilter-D1` trigger shape) is now covered by the `ArraySet 6` / `ArraySet 7` `SECTION`s of `Valid Threshold Sets`. | -| Tests today | **14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations / 31 ctest entries**, all passing in both the in-core and out-of-core Release builds (12 single-entry TEST_CASEs + 2 `TEMPLATE_TEST_CASE`s instantiated over 9 and 10 types respectively). Exhaustive sweeps over comparison operator × invert × union operator × set nesting × mask `DataType` (10 types, plus boolean covered elsewhere) × source-array `DataType` (11 types) × custom TRUE/FALSE execution (10 types), plus negative/error-path groups and a SIMPL backwards-compatibility check. All fixtures built in-memory. | +| Tests today | **14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations in `MultiThresholdObjectsVvTest.cpp`, with 31 CTest registrations in each 2026-10-12 Release build** (12 single-entry `TEST_CASE`s + 2 `TEMPLATE_TEST_CASE`s instantiated over 9 and 10 types respectively). The dated sign-off result remains below; the 2026-10-12 selected V&V run passed 31/31 in each build. Exhaustive sweeps over comparison operator × invert × union operator × set nesting × mask `DataType` (10 types, plus boolean covered elsewhere) × source-array `DataType` (11 types) × custom TRUE/FALSE execution (10 types), plus negative/error-path groups and a SIMPL backwards-compatibility check. All fixtures built in-memory. | | Exemplar archive | **None.** All fixtures are constructed in-memory by `CreateTestDataStructure()` / `CreateTestDataStructure2()`; no `.dream3d` exemplar or `download_test_data()` entry exists for this filter. | | Legacy comparison | **Run, and now complete — no configurations outstanding.** Independent three-way A/B (DREAM3D 6.5.171 `PipelineRunner` vs. this branch's `nxrunner` vs. an independent numpy oracle) across six configurations: flat/basic (`AB1`, vs. `MultiThresholdObjects`), nested and inverted-nested (`AB2`/`AB3`), and three deeper shapes added at second-engineer review — three-level nesting, an inverted sibling set, and a grouping-sensitive case (`AB4`–`AB6`, vs. `MultiThresholdObjects2`) — plus a 50M-tuple scale re-run of `AB1`–`AB3`. All sources MATCH in every case. Pre-fix (`develop`): `AB2`/`AB3` diverged (38/100 and 51/100 tuples wrong) — see `-D1`/`-D2`. `AB4`–`AB6` were additionally run through SIMPLNX's SIMPL-pipeline conversion, which surfaced `-D4`. | | Bug flags | **Three, all resolved, all quantified against real legacy output, all with in-repo regression coverage.** `-D1` — a set combining a leaf threshold with a sibling nested set produced an all-false mask (38/100 tuples wrong) on `develop`. `-D2` — an inverted nested set used `std::reverse` to flip tuple *order* instead of each tuple's value (51/100 wrong) on `develop`. `-D4` — **found at second-engineer review**: converting a legacy pipeline with a nested threshold set flattened the nesting and reset every union operator to AND, so converted `Threshold Objects (Advanced)` pipelines computed a plain AND over all leaves (55/100, 50/100 and 4/100 wrong on `AB4`–`AB6`); pre-existing on `develop`, in shared `ArrayThresholdsParameter` conversion code. See `vv/deviations/MultiThresholdObjectsFilter.md`. | @@ -24,7 +24,7 @@ ## Summary -`MultiThresholdObjectsFilter` builds a typed mask array by elementwise-comparing one or more input arrays against user-supplied thresholds, combined through an arbitrarily-nested tree of AND/OR/invert `ArrayThresholdSet`s. Verification uses a **Class 1 (Analytical) oracle**: every comparison operator, invert flag, union operator, nesting depth, custom TRUE/FALSE execution, and both the mask-output and source-input `DataType` are exhaustively hand-derived and asserted in `MultiThresholdObjectsTest.cpp` (14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations, all passing). 26 of 27 algorithm, preflight and SIMPL-conversion code paths are exercised. An independent three-way runtime A/B (legacy DREAM3D 6.5.171, this branch, and a numpy oracle) against both legacy predecessors — at 100 tuples and again at 50M tuples — confirms the current implementation matches legacy exactly, and quantifies two real bugs that were present on `develop` and are resolved: `MultiThresholdObjectsFilter-D1` (all-false mask when a set mixes a leaf threshold with a nested set, 38/100 tuples wrong) and `MultiThresholdObjectsFilter-D2` (`std::reverse`-based tuple-order corruption in an inverted nested set instead of per-value inversion, 51/100 tuples wrong). Both now have dedicated in-repo regression coverage (`Valid Threshold Sets` → `ArraySet 6` / `ArraySet 7`), each verified to fail when the algorithm is stubbed out. Second-engineer review extended the A/B to three deeper shapes (`AB4`–`AB6`) and to SIMPLNX's own SIMPL-pipeline conversion path, which surfaced a third bug, `MultiThresholdObjectsFilter-D4`: converting a legacy pipeline containing a nested threshold set flattened the nesting and reset every union operator to AND. Three further deviations (`-D3`, `-D5`, `-D6`) record SIMPLNX-only capabilities with no legacy equivalent — multi-component index selection, a selectable mask output `DataType`, and custom TRUE/FALSE values. +`MultiThresholdObjectsFilter` builds a typed mask array by elementwise-comparing one or more input arrays against user-supplied thresholds, combined through an arbitrarily-nested tree of AND/OR/invert `ArrayThresholdSet`s. Verification uses a **Class 1 (Analytical) oracle**: every comparison operator, invert flag, union operator, nesting depth, custom TRUE/FALSE execution, and both the mask-output and source-input `DataType` are exhaustively hand-derived and asserted in `MultiThresholdObjectsVvTest.cpp` (14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations; the dated sign-off records a 31/31 dual-build result). 26 of 27 algorithm, preflight and SIMPL-conversion code paths are exercised. An independent three-way runtime A/B (legacy DREAM3D 6.5.171, this branch, and a numpy oracle) against both legacy predecessors — at 100 tuples and again at 50M tuples — confirms the current implementation matches legacy exactly, and quantifies two real bugs that were present on `develop` and are resolved: `MultiThresholdObjectsFilter-D1` (all-false mask when a set mixes a leaf threshold with a nested set, 38/100 tuples wrong) and `MultiThresholdObjectsFilter-D2` (`std::reverse`-based tuple-order corruption in an inverted nested set instead of per-value inversion, 51/100 tuples wrong). Both now have dedicated in-repo regression coverage (`Valid Threshold Sets` → `ArraySet 6` / `ArraySet 7`), each verified to fail when the algorithm is stubbed out. Second-engineer review extended the A/B to three deeper shapes (`AB4`–`AB6`) and to SIMPLNX's own SIMPL-pipeline conversion path, which surfaced a third bug, `MultiThresholdObjectsFilter-D4`: converting a legacy pipeline containing a nested threshold set flattened the nesting and reset every union operator to AND. Three further deviations (`-D3`, `-D5`, `-D6`) record SIMPLNX-only capabilities with no legacy equivalent — multi-component index selection, a selectable mask output `DataType`, and custom TRUE/FALSE values. ## Algorithm Relationship @@ -68,7 +68,7 @@ *Comparison-value truncation (verified, legacy-matching):* the user-supplied comparison value is a `float64`, but `ThresholdFilterHelper::filterDataWithComparision` (`Algorithms/MultiThresholdObjects.cpp`) truncates it to the input array's type and performs the comparison in that type. A threshold of `5.5` against an `int32` array therefore compares against `5`, so `5 == 5.5` is **true** for this filter. This matches legacy DREAM3D exactly — `SIMPL/Source/SIMPLib/Filtering/ThresholdFilterHelper.h:60` performs the identical `static_cast(m_ComparisonValue)` — and it keeps 64-bit integer comparisons exact, which comparing in `float64` would not (values above 2^53 differing by 1 would collapse to equal). The `Expected*Mask` oracle helpers model the same truncation (`static_cast(thresholdValue)` for the int fixtures, `static_cast(...)` on both operands for the `Float32Array` fixture) so the oracle and the implementation agree by construction rather than by coincidence of fixture size. `Valid Single Thresholds: Int` deliberately `GENERATE`s a fractional threshold (`5.5`) against integer data to keep this behavior asserted. -*Encoded:* `test/MultiThresholdObjectsTest.cpp` — +*Encoded:* `test/MultiThresholdObjectsVvTest.cpp` — - `Exemplar Single Thresholds: Int` / `: Float` — single fixed-threshold fixtures checked against hardcoded `k_ExemplarInt4` / `k_ExemplarFloat02` arrays - `Valid Single Thresholds: Int` / `: Float` / `: Int Multi-Component` — comparison operator × invert × (component index for multi-component) sweep, via `ExpectedIntSingleComponentMask` / `ExpectedFloatSingleComponentMask` / `ExpectedIntMultiComponentMask` @@ -80,7 +80,7 @@ - `SIMPL Nested Set Conversion` — a converted legacy nested threshold set keeps its nesting, inversion flag and union operators - `Invalid Execution`, `Invalid Execution - Out of Bounds Custom Values` (9 numeric types), `Invalid Execution - Boolean Custom Values` — negative-path fixtures -14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations, all pass at HEAD. +The V&V file contains 14 `TEST_CASE`/`TEMPLATE_TEST_CASE` declarations. The 2026-08-19 sign-off result and a separate 2026-10-12 selected dual-build check are recorded below. *Second-engineer review:* **Signed off by Michael A. Jackson , 2026-08-19** (PR #1688 review, per sign-off convention). The V&V work was authored by Matthew Marine, so the review is independent of the author. Reviewed across two passes (2026-07-24 and 2026-08-12) plus this closing pass: @@ -153,13 +153,15 @@ Not counted as an algorithm/preflight path: the "Empty ArrayThreshold DataPath" **Missing:** nothing outstanding. The last gap — a set mixing a leaf threshold with a sibling nested set (row 25), the shape that triggered `MultiThresholdObjectsFilter-D1` — is closed by the `ArraySet 6` / `ArraySet 7` `SECTION`s of `Valid Threshold Sets`. -**Count basis:** 31 ctest entries = 12 single-entry `TEST_CASE`s (1 ctest entry each) + `Invalid Execution - Out of Bounds Custom Values` (`TEMPLATE_TEST_CASE`, 9 types → 9 entries) + `Valid Execution - Custom Values` (`TEMPLATE_TEST_CASE`, 10 types → 10 entries), verified by directly reading each declaration's type list twice. `catch_discover_tests` is called with no filtering options in `cmake/Plugin.cmake:404`, so Catch2's default behavior applies: one ctest entry per `TEMPLATE_TEST_CASE` type instantiation. A count of 28 entries cited during review was correct at the commit it was measured against; the two `Exemplar Single Thresholds` `TEST_CASE`s added afterwards account for the difference. 31 matches the test file as it stands today. +**V&V count basis (source-derived):** 31 CTest entries = 12 single-entry `TEST_CASE`s (1 ctest entry each) + `Invalid Execution - Out of Bounds Custom Values` (`TEMPLATE_TEST_CASE`, 9 types → 9 entries) + `Valid Execution - Custom Values` (`TEMPLATE_TEST_CASE`, 10 types → 10 entries), verified by directly reading each declaration's type list twice. `catch_discover_tests` is called with no filtering options in `cmake/Plugin.cmake:404`, so Catch2's default behavior applies: one ctest entry per `TEMPLATE_TEST_CASE` type instantiation. A count of 28 entries cited during review was correct at the commit it was measured against; the two `Exemplar Single Thresholds` `TEST_CASE`s added afterwards account for the difference. The 14 declarations are in `test/MultiThresholdObjectsVvTest.cpp`: 12 names begin `SimplnxCore::MultiThresholdObjects V&V:`, while the two SIMPL conversion names begin `SimplnxCore::MultiThresholdObjectsFilter V&V:`. `test/MultiThresholdObjectsTest.cpp` is a separate, broader filter suite and is excluded from this 14/31 V&V subset. The 2026-10-12 CTest discovery files confirm exactly these 31 V&V registrations in each Release build. -**Dual-build verification at sign-off:** `ctest -R MultiThresholdObjects` passes 31/31 in **both** builds — in-core (`NX-Com-Qt69-Vtk96-Rel`) and out-of-core (`NX-OOC-Qt69-Vtk95-Rel`). The full `SimplnxCore::` suite also passes 984/984 in-core. The OOC run initially reported all entries failing; all were `LoadPlugins()` aborting because stale plugin binaries in that build directory linked an older TBB soname, and all cleared after rebuilding those plugins — no failure was attributable to this filter. +**Dual-build verification recorded at the 2026-08-19 sign-off:** `ctest -R MultiThresholdObjects` passed 31/31 in **both** builds — in-core (`NX-Com-Qt69-Vtk96-Rel`) and out-of-core (`NX-OOC-Qt69-Vtk95-Rel`). The full `SimplnxCore::` suite also passed 984/984 in-core. The OOC run initially reported all entries failing; all were `LoadPlugins()` aborting because stale plugin binaries in that build directory linked an older TBB soname, and all cleared after rebuilding those plugins — no failure was attributable to this filter. + +**Selected current verification (2026-10-12):** CTest discovery found 31 names with the two V&V prefixes above in each Release build. The selected V&V subset passed 31/31 in `DREAM3DNX-Claude` (`SIMPLNX_TEST_ALGORITHM_PATH=0`, both algorithm paths) and 31/31 in `DREAM3DNX-OOC` (`SIMPLNX_TEST_ALGORITHM_PATH=1`, OOC algorithm path). The complete selected runs passed 33/33 and 69/69 respectively; those totals also include other filters' controls and are not V&V counts. The V&V fixtures use resident data and select in-memory store scenarios, so the OOC build result does not demonstrate disk-backed execution. This is a current selected-test check, not a new scientific V&V sign-off. Source heads: DREAM3DNX `27027499be134f92fdd22b029d52832dd6245fed`, simplnx `9a28a90f0f7a5d42fb5f17f36c84597b640fcac9`, SimplnxOoc `457d92b3b969a1d070a2d072492ecd14d8acbb79`. Discovery and run records: `DREAM3DNX/docs/ooc-audit-2026-09-28/supporting/optimization-implementation/closeout-baseline-{incore,ooc}-tests-{discovery,result}.json` and matching `-detail.log` files. ## Exemplar archive -None. All fixtures for this filter are constructed in-memory in `test/MultiThresholdObjectsTest.cpp` (`CreateTestDataStructure()`, `CreateTestDataStructure2()`); there is no `.dream3d` exemplar and no `download_test_data()` entry in `test/CMakeLists.txt` for `MultiThresholdObjectsFilter`. No provenance sidecar is needed. +None. All V&V fixtures are constructed in-memory in `test/MultiThresholdObjectsVvTest.cpp` (`CreateTestDataStructure()`, `CreateTestDataStructure2()`); there is no `.dream3d` exemplar and no `download_test_data()` entry in `test/CMakeLists.txt` for `MultiThresholdObjectsFilter`. No provenance sidecar is needed. ## Deviations from DREAM3D 6.5.171 diff --git a/src/Plugins/SimplnxCore/vv/deviations/ComputeFeatureCentroidsFilter.md b/src/Plugins/SimplnxCore/vv/deviations/ComputeFeatureCentroidsFilter.md index 780c554175..5868225568 100644 --- a/src/Plugins/SimplnxCore/vv/deviations/ComputeFeatureCentroidsFilter.md +++ b/src/Plugins/SimplnxCore/vv/deviations/ComputeFeatureCentroidsFilter.md @@ -44,6 +44,8 @@ Entries are referenced by stable ID (`ComputeFeatureCentroidsFilter-D`) from **Note (SIMPLNX-internal bugs fixed during V&V — not legacy deviations):** The periodic centroid had two SIMPLNX-internal defects, both fixed. (1) *Spacing:* the offset in the `ImageGeom` overload of `AdjustCentroidsForPeriodicFaces` was added in cell-index units (`(dim−1)/2`) to a physical-coordinate centroid, correct only when spacing == 1 (fixed during #1658). (2) *Model (issue #1665, 2026-07 follow-up):* the constant half-domain offset is itself only correct for a feature whose mass is symmetric about the seam, and can place asymmetric wrapped features outside the domain. The periodic path was reimplemented as a **minimum-image circular mean** accumulated during the cell sweep, and the range-based `ImageGeom` overload (insufficient inputs to compute a wrapped centroid) was removed. `test/ComputeFeatureCentroidsTest.cpp` Fixtures D/E (independent minimum-image oracle) and F (domain-filling fallback) pin the corrected behavior. Because 6.5.171 has no periodic path, neither fix is a change relative to legacy; recorded here for any SIMPLNX user who ran `Is Periodic` before the fixes. +**Current periodic behavior (2026-10 correction):** An adversarial unequal-population case showed that the July circular estimator did not implement the documented arithmetic mean after unwrapping. The current path chooses the unique largest empty gap among occupied cell centers and averages all unwrapped cells with their actual multiplicities. Tied largest gaps and complete axis occupancy retain the ordinary mean. The nonperiodic legacy comparison and the historical V&V sign-off above are unchanged. + --- ## Retracted candidate deviations diff --git a/src/Plugins/SimplnxCore/vv/provenance/ComputeFeatureCentroidsFilter.md b/src/Plugins/SimplnxCore/vv/provenance/ComputeFeatureCentroidsFilter.md index 3f89a4b4ce..f38da391c4 100644 --- a/src/Plugins/SimplnxCore/vv/provenance/ComputeFeatureCentroidsFilter.md +++ b/src/Plugins/SimplnxCore/vv/provenance/ComputeFeatureCentroidsFilter.md @@ -1,5 +1,7 @@ # Exemplar Provenance: ComputeFeatureCentroidsFilter +This sidecar records the original 2026-07 V&V fixtures and sign-off. The current 2026-10 periodic implementation and its independent weighted sorted-coordinate oracle are described in `../ComputeFeatureCentroidsFilter.md`; the historical periodic expected values below are not a description of every current periodic case. + **No exemplar archive.** As of this V&V cycle (2026-07-07) the filter's correctness tests are **inline Class 1 (Analytical) + Class 4 (Invariant) fixtures** built directly in `test/ComputeFeatureCentroidsTest.cpp` (`namespace CentroidToy`). There is no `.dream3d` gold-master to hash — the oracle lives in the test code as hand-derived `REQUIRE` values. ## Retired test / circular-oracle removal diff --git a/src/Plugins/TestOne/CMakeLists.txt b/src/Plugins/TestOne/CMakeLists.txt index 56c10cc86b..4d44bbed87 100644 --- a/src/Plugins/TestOne/CMakeLists.txt +++ b/src/Plugins/TestOne/CMakeLists.txt @@ -8,6 +8,7 @@ set(${PLUGIN_NAME}_SOURCE_DIR ${simplnx_SOURCE_DIR}/src/Plugins/${PLUGIN_NAME}) # PLUGIN_NAME/src/PLUGIN_NAME/Filters/ directory. set(FilterList ErrorWarningFilter + FlushFailureFilter ExampleFilter1Filter ExampleFilter2Filter # CreateOutOfCoreArray diff --git a/src/Plugins/TestOne/src/TestOne/Filters/FlushFailureFilter.cpp b/src/Plugins/TestOne/src/TestOne/Filters/FlushFailureFilter.cpp new file mode 100644 index 0000000000..ac7a024224 --- /dev/null +++ b/src/Plugins/TestOne/src/TestOne/Filters/FlushFailureFilter.cpp @@ -0,0 +1,248 @@ +#include "FlushFailureFilter.hpp" + +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/Common/Uuid.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/DataStructure/DataObject.hpp" +#include "simplnx/DataStructure/DataPath.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/Filter/Arguments.hpp" +#include "simplnx/Filter/ExecutionContext.hpp" +#include "simplnx/Filter/IFilter.hpp" +#include "simplnx/Filter/Parameters.hpp" +#include "simplnx/Parameters/NumberParameter.hpp" +#include "simplnx/Pipeline/Pipeline.hpp" +#include "simplnx/Pipeline/PipelineFilter.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +/** + * @class CliCompletionStore + * @brief Returns a selected checked error without requiring private storage code. + */ +class CliCompletionStore : public DataStore +{ +public: + /** + * @brief Creates the literal two-value carrier. + * @param mode Selects ordinary or allocation-category checked failure. + */ + explicit CliCompletionStore(uint64 mode) + : DataStore(ShapeType{2}, ShapeType{1}, int32{0}) + , m_Mode(mode) + { + setValue(0, 11); + setValue(1, 13); + } + + ~CliCompletionStore() override = default; + CliCompletionStore(const CliCompletionStore&) = default; + CliCompletionStore(CliCompletionStore&&) = default; + CliCompletionStore& operator=(const CliCompletionStore&) = delete; + CliCompletionStore& operator=(CliCompletionStore&&) = default; + + /** + * @brief Reports a checked-only completion fault while leaving legacy flush harmless. + * @return The selected checked result with a distinct warning. + */ + [[nodiscard]] Result<> flushChecked() const override + { + if(m_Mode == 1 || m_Mode == 2 || m_Mode == 3) + { + auto result = MakeErrorResult(m_Mode == 3 ? -272 : -76402, "C2-final-flush-error: cli-backing.h5:/Values"); + result.warnings().push_back({.code = 76403, .message = "C2-final-flush-warning"}); + return result; + } + return {}; + } + +private: + uint64 m_Mode; +}; + +/** + * @struct CliSnapshotControl + * @brief Retains the End connection without a cycle to its owning data objects. + */ +struct CliSnapshotControl +{ + bool Armed = false; + bool Allocation = false; + nod::scoped_connection EndConnection; +}; + +/** + * @class CliSnapshotGroup + * @brief Throws only during an armed shallow-copy operation. + */ +class CliSnapshotGroup : public DataGroup +{ +public: + /** + * @brief Creates an insertable group with shared fault state. + * @param dataStructure Owns the group after insertion. + * @param control Arms an ordinary or simulated allocation copy error. + */ + CliSnapshotGroup(DataStructure& dataStructure, std::shared_ptr control) + : DataGroup(dataStructure, "CompletionSnapshot") + , m_Control(std::move(control)) + { + } + + /** + * @brief Preserves the control in an unarmed shallow copy. + * @param other Supplies identity and the shared control. + */ + CliSnapshotGroup(const CliSnapshotGroup& other) = default; + CliSnapshotGroup(CliSnapshotGroup&&) = delete; + CliSnapshotGroup& operator=(const CliSnapshotGroup&) = delete; + CliSnapshotGroup& operator=(CliSnapshotGroup&&) = delete; + ~CliSnapshotGroup() override = default; + + /** + * @brief Copies the group or raises the selected snapshot exception. + * @return A caller-owned shallow copy when unarmed. + * @throws std::bad_alloc If an allocation fault is armed. + * @throws std::runtime_error If an ordinary fault is armed. + */ + DataObject* shallowCopy() override + { + if(m_Control->Armed) + { + if(m_Control->Allocation) + { + throw std::bad_alloc{}; + } + throw std::runtime_error("C2-copy-error: CLI completion snapshot"); + } + return new CliSnapshotGroup(*this); + } + +private: + std::shared_ptr m_Control; +}; +} // namespace + +std::string FlushFailureFilter::name() const +{ + return FilterTraits::name; +} +std::string FlushFailureFilter::className() const +{ + return FilterTraits::className; +} +Uuid FlushFailureFilter::uuid() const +{ + return FilterTraits::uuid; +} +std::string FlushFailureFilter::humanName() const +{ + return "Completion Failure Test Fixture"; +} + +//------------------------------------------------------------------------------ +std::vector FlushFailureFilter::defaultTags() const +{ + return {className(), "Example", "Test"}; +} + +IFilter::VersionType FlushFailureFilter::parametersVersion() const +{ + return 1; +} + +Parameters FlushFailureFilter::parameters() const +{ + Parameters result; + result.insert(std::make_unique( + k_Mode_Key, "Completion Test Mode", "0: unarmed; 1: flush error; 2: execute and flush errors; 3: flush allocation; 4: repeated snapshot; 5: root snapshot; 6: snapshot allocation.", uint64{0})); + return result; +} + +IFilter::UniquePointer FlushFailureFilter::clone() const +{ + return std::make_unique(); +} + +IFilter::PreflightResult FlushFailureFilter::preflightImpl([[maybe_unused]] const DataStructure& dataStructure, const Arguments& filterArgs, [[maybe_unused]] const MessageHandler& messageHandler, + [[maybe_unused]] const std::atomic_bool& shouldCancel, [[maybe_unused]] const ExecutionContext& executionContext) const +{ + const auto mode = filterArgs.value(k_Mode_Key); + if(mode > 6) + { + return MakePreflightErrorResult(-76409, fmt::format("Completion Test Mode {} is outside the fixture range [0, 6].", mode)); + } + PreflightResult result{}; + result.outputValues.push_back({.name = "Completion value", .value = "preserved"}); + return result; +} + +Result<> FlushFailureFilter::executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, + [[maybe_unused]] const std::atomic_bool& shouldCancel, [[maybe_unused]] const ExecutionContext& executionContext) const +{ + const auto mode = filterArgs.value(k_Mode_Key); + messageHandler.sendInfoMessage("C2 fixture UUID: 2a98fc79-c485-45d5-8fa9-1f298f3d6729"); + auto store = std::make_shared(mode); + if(DataArray::Create(dataStructure, "Values", store) == nullptr) + { + return MakeErrorResult(-76409, "Cannot insert CLI completion fixture array 'Values'."); + } + if(mode >= 4) + { + auto control = std::make_shared(); + control->Allocation = mode == 6; + control->Armed = mode != 5; + if(!dataStructure.insert(std::make_shared(dataStructure, control), DataPath{})) + { + return MakeErrorResult(-76409, "Cannot insert CLI completion fixture group 'CompletionSnapshot'."); + } + if(mode == 5) + { + auto* parentPtr = pipelineNode == nullptr ? nullptr : pipelineNode->getParentPipeline(); + bool connected = false; + if(parentPtr != nullptr) + { + for(const auto& node : *parentPtr) + { + if(node.get() == pipelineNode) + { + const std::weak_ptr weakControl = control; + control->EndConnection = node->getFilterUpdateSignal().connect([weakControl](AbstractPipelineNode*, int32, const std::string& message) { + if(message == "End") + { + if(auto state = weakControl.lock()) + { + state->Armed = true; + } + } + }); + connected = true; + break; + } + } + } + if(!connected) + { + return MakeErrorResult(-76409, "Root snapshot fixture requires its actual parent PipelineFilter."); + } + } + } + auto result = mode == 2 ? MakeErrorResult(-76401, "C2-original-execute-error") : Result<>{}; + result.warnings().push_back({.code = 76400, .message = "C2-original-execute-warning"}); + return result; +} diff --git a/src/Plugins/TestOne/src/TestOne/Filters/FlushFailureFilter.hpp b/src/Plugins/TestOne/src/TestOne/Filters/FlushFailureFilter.hpp new file mode 100644 index 0000000000..37693cbbf1 --- /dev/null +++ b/src/Plugins/TestOne/src/TestOne/Filters/FlushFailureFilter.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include "TestOne/TestOne_export.hpp" +#include "simplnx/Filter/FilterTraits.hpp" +#include "simplnx/Filter/IFilter.hpp" + +namespace nx::core +{ +/** + * @class FlushFailureFilter + * @brief Supplies bounded completion faults for actual command-line acceptance tests. + */ +class TESTONE_EXPORT FlushFailureFilter : public IFilter +{ +public: + static constexpr StringLiteral k_Mode_Key = "completion_test_mode"; + + /** + * @brief Creates an unarmed test fixture filter. + */ + FlushFailureFilter() = default; + ~FlushFailureFilter() noexcept override = default; + + FlushFailureFilter(const FlushFailureFilter&) = delete; + FlushFailureFilter(FlushFailureFilter&&) = delete; + FlushFailureFilter& operator=(const FlushFailureFilter&) = delete; + FlushFailureFilter& operator=(FlushFailureFilter&&) = delete; + + std::string name() const override; + std::string className() const override; + Uuid uuid() const override; + std::string humanName() const override; + VersionType parametersVersion() const override; + std::vector defaultTags() const override; + + /** + * @brief Declares the bounded test-mode parameter. + * @return A parameter set with unarmed mode as its default. + */ + Parameters parameters() const override; + + /** + * @brief Creates a fresh filter; arguments carry the selected fixture mode. + * @return A new filter instance. + */ + UniquePointer clone() const override; + +protected: + /** + * @brief Rejects unknown fixture modes before execution. + * @param dataStructure Is unchanged during preflight. + * @param filterArgs Selects a mode from zero through six. + * @param messageHandler Is unused during preflight. + * @param shouldCancel Is checked by the IFilter wrapper. + * @param executionContext Is supplied by the parent pipeline. + * @return Valid actions and a sentinel, or an invalid-mode error. + */ + PreflightResult preflightImpl(const DataStructure& dataStructure, const Arguments& filterArgs, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel, + const ExecutionContext& executionContext) const override; + + /** + * @brief Installs literal data and arms the selected completion fault. + * @param dataStructure Receives the small resident fixture and optional copy-fault group. + * @param filterArgs Selects the completion mode. + * @param pipelineNode Identifies the node for the root-only End boundary. + * @param messageHandler Reports the executed fixture UUID. + * @param shouldCancel Is checked by the IFilter wrapper. + * @param executionContext Is supplied by the parent pipeline. + * @return An original execute error only in the combined-failure mode. + */ + Result<> executeImpl(DataStructure& dataStructure, const Arguments& filterArgs, const PipelineFilter* pipelineNode, const MessageHandler& messageHandler, const std::atomic_bool& shouldCancel, + const ExecutionContext& executionContext) const override; +}; +} // namespace nx::core + +SIMPLNX_DEF_FILTER_TRAITS(nx::core, FlushFailureFilter, "2a98fc79-c485-45d5-8fa9-1f298f3d6729"); diff --git a/src/Plugins/TestOne/test/CMakeLists.txt b/src/Plugins/TestOne/test/CMakeLists.txt index 9a8dbc5c1f..92c708be16 100644 --- a/src/Plugins/TestOne/test/CMakeLists.txt +++ b/src/Plugins/TestOne/test/CMakeLists.txt @@ -35,3 +35,13 @@ create_simplnx_plugin_unit_test(PLUGIN_NAME ${PLUGIN_NAME} PUBLIC ${simplnx_SOURCE_DIR}/test ) + +target_sources(${PLUGIN_NAME}UnitTest PRIVATE "${simplnx_SOURCE_DIR}/test/ProfileIsolationContract.cpp") + +add_test(NAME "${PLUGIN_NAME}_test_profile_isolation" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -DPROFILE_RUNNER_ONLY=TRUE + -P "${simplnx_SOURCE_DIR}/test/VerifyTestProfileLifecycle.cmake") +set_tests_properties("${PLUGIN_NAME}_test_profile_isolation" PROPERTIES RUN_SERIAL TRUE TIMEOUT 60) diff --git a/src/Plugins/TestOne/test/TestOne_test_main.cpp b/src/Plugins/TestOne/test/TestOne_test_main.cpp index 48425c5326..6b322c3f17 100644 --- a/src/Plugins/TestOne/test/TestOne_test_main.cpp +++ b/src/Plugins/TestOne/test/TestOne_test_main.cpp @@ -1,4 +1,11 @@ // Catch2 recommends placing these lines by themselves in a translation unit // which will help reduce unnecessary recompilations of the expensive Catch main -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_RUNNER #include + +#include "simplnx/UnitTest/OwnedCatchRunner.hpp" + +int main(int argc, char* argv[]) +{ + return nx::core::UnitTest::RunOwnedCatchSession(argc, argv); +} diff --git a/src/nxrunner/CMakeLists.txt b/src/nxrunner/CMakeLists.txt index f8bb2ecd87..bd8e0f7065 100644 --- a/src/nxrunner/CMakeLists.txt +++ b/src/nxrunner/CMakeLists.txt @@ -34,6 +34,18 @@ include(${simplnx_SOURCE_DIR}/cmake/Utility.cmake) simplnx_enable_warnings(TARGET nxrunner) if(BUILD_TESTING) + add_executable(P1PreferencesDefaultsProbe test/PreferencesDefaultsProbe.cpp) + target_link_libraries(P1PreferencesDefaultsProbe PRIVATE simplnx::simplnx) + simplnx_enable_warnings(TARGET P1PreferencesDefaultsProbe) + add_test(NAME "nxrunner OOC threshold preferences" + COMMAND ${CMAKE_COMMAND} + "-DNXRUNNER_EXECUTABLE=$" + "-DDEFAULTS_PROBE=$" + "-DPIPELINE_FILE=${CMAKE_CURRENT_SOURCE_DIR}/test/ooc_failure_pipeline.d3dpipeline" + "-DP1_ROOT_BASE=${CMAKE_CURRENT_BINARY_DIR}/P1ThresholdProfiles" + -P "${CMAKE_CURRENT_SOURCE_DIR}/test/OocThresholdPreferencesTest.cmake") + set_tests_properties("nxrunner OOC threshold preferences" PROPERTIES RUN_SERIAL TRUE TIMEOUT 750) + add_test( NAME "nxrunner cache memory budget CLI" COMMAND ${CMAKE_COMMAND} diff --git a/src/nxrunner/src/CliObserver.cpp b/src/nxrunner/src/CliObserver.cpp index 3f8ba69938..70253c0ca6 100644 --- a/src/nxrunner/src/CliObserver.cpp +++ b/src/nxrunner/src/CliObserver.cpp @@ -1,4 +1,6 @@ #include "CliObserver.hpp" +#include "simplnx/Common/Result.hpp" +#include #include "simplnx/Pipeline/Messaging/AbstractPipelineMessage.hpp" #include "simplnx/Utilities/TimeUtilities.hpp" @@ -22,6 +24,24 @@ PipelineObserver::PipelineObserver(Pipeline* pipeline) { return; } + m_SignalConnections.emplace_back(pipeline->getFilterFaultDetailSignal().connect([](AbstractPipelineNode*, int32_t, const WarningCollection& warnings, const ErrorCollection& errors) { + if(!warnings.empty()) + { + std::cout << "Pipeline completion warnings" << std::endl; + } + for(const auto& warning : warnings) + { + std::cout << " Code: " << warning.code << " Message: " << warning.message << std::endl; + } + if(!errors.empty()) + { + std::cout << "Pipeline completion errors" << std::endl; + } + for(const auto& error : errors) + { + std::cout << " Code: " << error.code << " Message: " << error.message << std::endl; + } + })); int32_t currentFilterIndex = 0; for(const auto& cxFilter : *pipeline) { diff --git a/src/nxrunner/src/nxrunner.cpp b/src/nxrunner/src/nxrunner.cpp index 77857c3f49..e5ddff96dd 100644 --- a/src/nxrunner/src/nxrunner.cpp +++ b/src/nxrunner/src/nxrunner.cpp @@ -423,7 +423,8 @@ Result<> ExecutePipeline(const Argument& arg) } if(!loadPipelineResult.m_Warnings.empty()) { - cliOut << "Input Pipeline Warnings" << "\n"; + cliOut << "Input Pipeline Warnings" + << "\n"; for(const auto& warning : loadPipelineResult.m_Warnings) { cliOut << fmt::format(" [{}] {}", warning.code, warning.message) << "\n"; @@ -467,7 +468,8 @@ Result<> PreflightPipeline(const Argument& arg) } if(!loadPipelineResult.m_Warnings.empty()) { - cliOut << "Preflight Pipeline: Input Pipeline Warnings" << "\n"; + cliOut << "Preflight Pipeline: Input Pipeline Warnings" + << "\n"; for(const auto& warning : loadPipelineResult.m_Warnings) { cliOut << fmt::format(" [{}] {}", warning.code, warning.message) << "\n"; @@ -771,6 +773,10 @@ int main(int argc, char* argv[]) // Create the Application before the temporary preference override. auto app = nx::core::Application::GetOrCreateInstance(); + for(const auto& warning : app->takePreferenceLoadWarnings()) + { + fmt::print(stderr, "Warning {}: {}\n", warning.code, warning.message); + } /** * @struct PreferencesCacheBudgetRestorer diff --git a/src/nxrunner/test/OocThresholdPreferencesTest.cmake b/src/nxrunner/test/OocThresholdPreferencesTest.cmake new file mode 100644 index 0000000000..8ca3afbc92 --- /dev/null +++ b/src/nxrunner/test/OocThresholdPreferencesTest.cmake @@ -0,0 +1,194 @@ +cmake_minimum_required(VERSION 3.26) + +foreach(required NXRUNNER_EXECUTABLE DEFAULTS_PROBE PIPELINE_FILE) + if(NOT DEFINED ${required} OR NOT EXISTS "${${required}}") + message(FATAL_ERROR "${required} was not supplied as an existing path") + endif() +endforeach() +if(NOT DEFINED P1_ROOT_BASE OR "${P1_ROOT_BASE}" STREQUAL "" OR NOT IS_ABSOLUTE "${P1_ROOT_BASE}") + message(FATAL_ERROR "P1_ROOT_BASE must be supplied as a nonempty absolute path; the directory need not exist") +endif() +include("${CMAKE_CURRENT_LIST_DIR}/OocProcessExitStatus.cmake") + +execute_process(COMMAND "${DEFAULTS_PROBE}" RESULT_VARIABLE probe_status OUTPUT_VARIABLE defaults_output TIMEOUT 30) +if(NOT probe_status STREQUAL "0") + message(FATAL_ERROR "The local Preferences default probe failed: ${probe_status}") +endif() +string(REGEX MATCHALL "[0-9]+" defaults "${defaults_output}") +list(LENGTH defaults default_count) +if(NOT default_count EQUAL 2) + message(FATAL_ERROR "The local Preferences default probe did not return exactly two byte counts: ${defaults_output}") +endif() +list(GET defaults 0 array_default) +list(GET defaults 1 structure_default) +set(expected_array_line "Warning -591: Preferences: saved Large Data Size ('large_data_size') value -1 (JSON type number_integer) is invalid. Allowed domain: 0 to 18446744073709551615 bytes. Using the default: ${array_default} bytes. Review this value in Preferences > Out of Core.") +set(expected_structure_line "Warning -591: Preferences: saved Large Data Structure Size ('large_datastructure_size') value \"bad\" (JSON type string) is invalid. Allowed domain: 0 to 18446744073709551615 bytes. Using the default: ${structure_default} bytes. Review this value in Preferences > Out of Core.") + +# Formats a byte count as GB with two decimals (1 GB = 1024^3 bytes), rounded to the nearest 0.01 GB. +# A nonzero byte count that rounds to zero becomes 0.01 GB. +function(format_bytes_as_gb bytes output_variable) + math(EXPR whole_gib "${bytes} / 1073741824") + math(EXPR remainder_bytes "${bytes} % 1073741824") + math(EXPR hundredths "${whole_gib} * 100 + (${remainder_bytes} * 100 + 536870912) / 1073741824") + if(hundredths EQUAL 0 AND NOT bytes EQUAL 0) + set(hundredths 1) + endif() + math(EXPR whole_part "${hundredths} / 100") + math(EXPR fraction_part "${hundredths} % 100") + if(fraction_part LESS 10) + set(fraction_part "0${fraction_part}") + endif() + set(${output_variable} "${whole_part}.${fraction_part}" PARENT_SCOPE) +endfunction() +format_bytes_as_gb("${array_default}" array_default_gb) +format_bytes_as_gb("${structure_default}" structure_default_gb) +set(expected_array_gb_line "Warning -592: Preferences: saved Large Data Size ('large_data_size_gb') value -1 (JSON type number_integer) is invalid. Allowed domain: 0 to 17179869183.99 GB. Using the default: ${array_default_gb} GB. Review this value in Preferences > Out of Core.") +set(expected_structure_gb_line "Warning -592: Preferences: saved Large Data Structure Size ('large_datastructure_size_gb') value \"bad\" (JSON type string) is invalid. Allowed domain: 0 to 17179869183.99 GB. Using the default: ${structure_default_gb} GB. Review this value in Preferences > Out of Core.") + +file(READ "${PIPELINE_FILE}" pipeline_template) +string(JSON source_format GET "${pipeline_template}" pipeline 1 args data_format value) +if(NOT source_format STREQUAL "HDF5-OOC") + message(FATAL_ERROR "The preflight fixture's pipeline[1].args.data_format.value changed") +endif() +string(REGEX MATCHALL "\"value\"[ \t]*:[ \t]*\"HDF5-OOC\"" format_tokens "${pipeline_template}") +list(LENGTH format_tokens format_count) +if(NOT format_count EQUAL 1) + message(FATAL_ERROR "The preflight fixture does not have one exact format token") +endif() +string(REPLACE "\"value\": \"HDF5-OOC\"" "\"value\": \"\"" pipeline "${pipeline_template}") +string(JSON selected_format GET "${pipeline}" pipeline 1 args data_format value) +if(NOT selected_format STREQUAL "") + message(FATAL_ERROR "The owned preflight copy did not clear the explicit format") +endif() +string(RANDOM LENGTH 12 ALPHABET 0123456789abcdef nonce) + +foreach(profile_case IN ITEMS clean one_invalid both_invalid zero maximum gb_valid gb_invalid) + set(case_root "${P1_ROOT_BASE}/${profile_case}-${nonce}") + file(MAKE_DIRECTORY "${case_root}/temp" "${case_root}/AppData/Local/DREAM3DNX" "${case_root}/AppData/Roaming" "${case_root}/Library/Preferences/DREAM3DNX" "${case_root}/.config/DREAM3DNX") + if(WIN32) + set(preferences_file "${case_root}/AppData/Local/DREAM3DNX/preferences.json") + elseif(APPLE) + set(preferences_file "${case_root}/Library/Preferences/DREAM3DNX/preferences.json") + else() + set(preferences_file "${case_root}/.config/DREAM3DNX/preferences.json") + endif() + if(profile_case STREQUAL "clean") + set(profile "{\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 0) + elseif(profile_case STREQUAL "one_invalid") + set(profile "{\"large_data_size\":-1,\"large_datastructure_size\":18446744073709551615,\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 1) + elseif(profile_case STREQUAL "both_invalid") + set(profile "{\"large_data_size\":-1,\"large_datastructure_size\":\"bad\",\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 2) + elseif(profile_case STREQUAL "zero") + set(profile "{\"large_data_size\":0,\"large_datastructure_size\":0,\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 0) + elseif(profile_case STREQUAL "maximum") + set(profile "{\"large_data_size\":18446744073709551615,\"large_datastructure_size\":18446744073709551615,\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 0) + elseif(profile_case STREQUAL "gb_valid") + set(profile "{\"large_data_size_gb\":1.5,\"large_datastructure_size_gb\":0.01,\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 0) + else() + set(profile "{\"large_data_size_gb\":-1,\"large_datastructure_size_gb\":\"bad\",\"plugins\":{\"P1\":{\"keep\":17}}}") + set(expected_count 2) + endif() + file(WRITE "${preferences_file}" "${profile}") + file(WRITE "${case_root}/input-preferences.json" "${profile}") + file(WRITE "${case_root}/pipeline.d3dpipeline" "${pipeline}") + + set(ENV{HOME} "${case_root}") + set(ENV{USERPROFILE} "${case_root}") + set(ENV{LOCALAPPDATA} "${case_root}/AppData/Local") + set(ENV{APPDATA} "${case_root}/AppData/Roaming") + set(ENV{TEMP} "${case_root}/temp") + set(ENV{TMP} "${case_root}/temp") + set(ENV{TMPDIR} "${case_root}/temp") + unset(ENV{SIMPLNX_OOC_FAULT_INJECT}) + execute_process(COMMAND "${NXRUNNER_EXECUTABLE}" --preflight "${case_root}/pipeline.d3dpipeline" + WORKING_DIRECTORY "${case_root}" RESULT_VARIABLE run_status OUTPUT_FILE "${case_root}/stdout.txt" + ERROR_FILE "${case_root}/stderr.txt" TIMEOUT 120) + simplnx_classify_ooc_process_status("${run_status}" classification) + if(NOT classification STREQUAL "SUCCESS") + message(FATAL_ERROR "${profile_case} preflight failed with '${run_status}'; evidence: ${case_root}") + endif() + file(READ "${case_root}/stdout.txt" stdout) + file(READ "${case_root}/stderr.txt" stderr) + string(REGEX MATCHALL "Warning -59[12]: [^\r\n]+" corrections "${stderr}") + list(LENGTH corrections correction_count) + if(NOT correction_count EQUAL expected_count) + message(FATAL_ERROR "${profile_case}: expected ${expected_count} corrections, got ${correction_count}; evidence: ${case_root}") + endif() + if(stdout MATCHES "Warning -59[12]:") + message(FATAL_ERROR "${profile_case}: correction appeared on stdout; evidence: ${case_root}") + endif() + set(array_corrections 0) + set(structure_corrections 0) + foreach(correction IN LISTS corrections) + string(STRIP "${correction}" correction) + # Test the GB keys first: "large_data_size" is also part of "large_data_size_gb". + if(correction MATCHES "large_data_size_gb") + math(EXPR array_corrections "${array_corrections}+1") + if(NOT correction STREQUAL expected_array_gb_line) + message(FATAL_ERROR "${profile_case}: array GB correction differs from the exact core message; evidence: ${case_root}\n${correction}") + endif() + elseif(correction MATCHES "large_datastructure_size_gb") + math(EXPR structure_corrections "${structure_corrections}+1") + if(NOT correction STREQUAL expected_structure_gb_line) + message(FATAL_ERROR "${profile_case}: structure GB correction differs from the exact core message; evidence: ${case_root}\n${correction}") + endif() + elseif(correction MATCHES "large_data_size") + math(EXPR array_corrections "${array_corrections}+1") + if(NOT correction STREQUAL expected_array_line) + message(FATAL_ERROR "${profile_case}: array correction differs from the exact core message; evidence: ${case_root}\n${correction}") + endif() + elseif(correction MATCHES "large_datastructure_size") + math(EXPR structure_corrections "${structure_corrections}+1") + if(NOT correction STREQUAL expected_structure_line) + message(FATAL_ERROR "${profile_case}: structure correction differs from the exact core message; evidence: ${case_root}\n${correction}") + endif() + else() + message(FATAL_ERROR "${profile_case}: unexpected correction key; evidence: ${case_root}") + endif() + endforeach() + if(profile_case STREQUAL "one_invalid" AND NOT array_corrections EQUAL 1) + message(FATAL_ERROR "${profile_case}: expected one array correction; evidence: ${case_root}") + endif() + if((profile_case STREQUAL "both_invalid" OR profile_case STREQUAL "gb_invalid") AND (NOT array_corrections EQUAL 1 OR NOT structure_corrections EQUAL 1)) + message(FATAL_ERROR "${profile_case}: expected one correction for each field; evidence: ${case_root}") + endif() + if(profile_case STREQUAL "both_invalid" OR profile_case STREQUAL "gb_invalid") + list(GET corrections 0 first_correction) + list(GET corrections 1 second_correction) + if(NOT first_correction MATCHES "large_data_size" OR NOT second_correction MATCHES "large_datastructure_size") + message(FATAL_ERROR "${profile_case}: correction order changed; evidence: ${case_root}") + endif() + endif() + file(READ "${preferences_file}" saved) + string(JSON kept GET "${saved}" plugins P1 keep) + if(NOT kept STREQUAL "17") + message(FATAL_ERROR "${profile_case}: unrelated plugin value changed; evidence: ${case_root}") + endif() + # The file stores the thresholds in GB (1 GB = 1024^3 bytes) and never keeps a byte key. + if(saved MATCHES "\"large_data_size\"[ \t]*:" OR saved MATCHES "\"large_datastructure_size\"[ \t]*:") + message(FATAL_ERROR "${profile_case}: a byte threshold key was saved; evidence: ${case_root}") + endif() + if(profile_case STREQUAL "one_invalid" OR profile_case STREQUAL "maximum") + if(NOT saved MATCHES "\"large_datastructure_size_gb\"[ \t]*:[ \t]*17179869183\\.99[ \t]*[,}]") + message(FATAL_ERROR "${profile_case}: UINT64_MAX did not convert to 17179869183.99 GB; evidence: ${case_root}") + endif() + endif() + if(profile_case STREQUAL "maximum" AND NOT saved MATCHES "\"large_data_size_gb\"[ \t]*:[ \t]*17179869183\\.99[ \t]*[,}]") + message(FATAL_ERROR "${profile_case}: per-array UINT64_MAX did not convert to 17179869183.99 GB; evidence: ${case_root}") + endif() + if(profile_case STREQUAL "zero" AND (NOT saved MATCHES "\"large_data_size_gb\"[ \t]*:[ \t]*0\\.0[ \t]*[,}]" OR NOT saved MATCHES "\"large_datastructure_size_gb\"[ \t]*:[ \t]*0\\.0[ \t]*[,}]")) + message(FATAL_ERROR "${profile_case}: zero threshold did not survive normal saving; evidence: ${case_root}") + endif() + if(profile_case STREQUAL "gb_valid" AND (NOT saved MATCHES "\"large_data_size_gb\"[ \t]*:[ \t]*1\\.5[ \t]*[,}]" OR NOT saved MATCHES "\"large_datastructure_size_gb\"[ \t]*:[ \t]*0\\.01[ \t]*[,}]")) + message(FATAL_ERROR "${profile_case}: saved GB thresholds changed; evidence: ${case_root}") + endif() + if((profile_case STREQUAL "both_invalid" OR profile_case STREQUAL "gb_invalid") AND (saved MATCHES "large_data_size_gb" OR saved MATCHES "large_datastructure_size_gb")) + message(FATAL_ERROR "${profile_case}: a repaired threshold was saved instead of its default; evidence: ${case_root}") + endif() +endforeach() diff --git a/src/nxrunner/test/PipelineFlushReachesUserTest.cmake b/src/nxrunner/test/PipelineFlushReachesUserTest.cmake new file mode 100644 index 0000000000..8698bf8d07 --- /dev/null +++ b/src/nxrunner/test/PipelineFlushReachesUserTest.cmake @@ -0,0 +1,151 @@ +# Execute the real CLI with a registered test-only filter and, when applicable, +# the real WriteDREAM3DFilter. All writable paths belong to this test's scratch directory. +foreach(required IN ITEMS NXRUNNER_EXECUTABLE TEST_MODE SCRATCH_ROOT) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "${required} was not provided") + endif() +endforeach() +if(NOT EXISTS "${NXRUNNER_EXECUTABLE}") + message(FATAL_ERROR "nxrunner executable is missing: ${NXRUNNER_EXECUTABLE}") +endif() +set(modes unarmed flush execute_and_flush allocation snapshot root_snapshot snapshot_allocation) +list(FIND modes "${TEST_MODE}" mode_index) +if(mode_index LESS 0) + message(FATAL_ERROR "Unknown completion test mode: ${TEST_MODE}") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/OocProcessExitStatus.cmake") +file(MAKE_DIRECTORY + "${SCRATCH_ROOT}/home" + "${SCRATCH_ROOT}/temp" + "${SCRATCH_ROOT}/home/AppData/Local" + "${SCRATCH_ROOT}/home/AppData/Roaming" + "${SCRATCH_ROOT}/home/.config" + "${SCRATCH_ROOT}/home/.cache" + "${SCRATCH_ROOT}/home/.local/share" + "${SCRATCH_ROOT}/home/.local/state" +) +file(TO_CMAKE_PATH "${SCRATCH_ROOT}/export.dream3d" export_path) +file(TO_CMAKE_PATH "${SCRATCH_ROOT}/pipeline.d3dpipeline" pipeline_path) +file(WRITE "${export_path}" "C2-owned-export-sentinel") +file(SHA256 "${export_path}" old_export_hash) +set(fixture_uuid "2a98fc79-c485-45d5-8fa9-1f298f3d6729") +set(first_filter [=[ +{ + "filter": {"name": "nx::core::FlushFailureFilter", "uuid": "@fixture_uuid@"}, + "args": {"parameters_version": 1, "completion_test_mode": {"version": 1, "value": @mode_index@}}, + "comments": "", + "isDisabled": false +} +]=]) +string(CONFIGURE "${first_filter}" first_filter @ONLY) +set(writer_filter [=[ +{ + "filter": {"name": "nx::core::WriteDREAM3DFilter", "uuid": "b3a95784-2ced-41ec-8d3d-0242ac130003"}, + "args": { + "parameters_version": 2, + "export_file_path": {"version": 1, "value": "@export_path@"}, + "write_xdmf_file": {"version": 1, "value": false}, + "use_compression": {"version": 1, "value": false}, + "compression_level": {"version": 1, "value": 5} + }, + "comments": "", + "isDisabled": false +} +]=]) +string(CONFIGURE "${writer_filter}" writer_filter @ONLY) +# Root-only failure happens after all filters; no test can promise rollback of a prior writer. +if(TEST_MODE STREQUAL "root_snapshot") + set(pipeline_filters "${first_filter}") +else() + set(pipeline_filters "${first_filter},${writer_filter}") +endif() +file(WRITE "${pipeline_path}" "{\"version\":1,\"name\":\"C2 CLI completion\",\"isDisabled\":false,\"pipeline\":[${pipeline_filters}]}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env --unset=SIMPLNX_OOC_FAULT_INJECT + "HOME=${SCRATCH_ROOT}/home" "USERPROFILE=${SCRATCH_ROOT}/home" + "LOCALAPPDATA=${SCRATCH_ROOT}/home/AppData/Local" "APPDATA=${SCRATCH_ROOT}/home/AppData/Roaming" + "TEMP=${SCRATCH_ROOT}/temp" "TMP=${SCRATCH_ROOT}/temp" "TMPDIR=${SCRATCH_ROOT}/temp" + "XDG_CONFIG_HOME=${SCRATCH_ROOT}/home/.config" "XDG_CACHE_HOME=${SCRATCH_ROOT}/home/.cache" + "XDG_DATA_HOME=${SCRATCH_ROOT}/home/.local/share" "XDG_STATE_HOME=${SCRATCH_ROOT}/home/.local/state" + "${NXRUNNER_EXECUTABLE}" --execute "${pipeline_path}" + RESULT_VARIABLE run_result + OUTPUT_VARIABLE run_stdout + ERROR_VARIABLE run_stderr + TIMEOUT 90 +) +set(output "${run_stdout}${run_stderr}") +simplnx_classify_ooc_process_status("${run_result}" classification) +if(classification STREQUAL "ABNORMAL") + message(FATAL_ERROR "Completion run ended abnormally (${run_result}); unknown-filter, exception, crash, and timeout exits are not expected failure:\n${output}") +endif() +string(FIND "${output}" "C2 fixture UUID: ${fixture_uuid}" fixture_witness) +if(fixture_witness LESS 0) + message(FATAL_ERROR "The actual registered fixture did not execute. An unknown-filter failure is not completion evidence:\n${output}") +endif() +if(TEST_MODE STREQUAL "unarmed") + if(NOT classification STREQUAL "SUCCESS") + message(FATAL_ERROR "Unarmed control failed with ${run_result}:\n${output}") + endif() + file(READ "${export_path}" signature OFFSET 0 LIMIT 8 HEX) + if(NOT signature STREQUAL "894844460d0a1a0a") + message(FATAL_ERROR "The real writer did not publish an HDF5 file: ${signature}\n${output}") + endif() + message(STATUS "Unarmed completion fixture and real DREAM3D writer succeeded.") + return() +endif() +if(NOT classification STREQUAL "EXPECTED_FAILURE") + message(FATAL_ERROR "Armed completion run returned success:\n${output}") +endif() +if(NOT TEST_MODE STREQUAL "root_snapshot") + file(SHA256 "${export_path}" export_hash) + if(NOT export_hash STREQUAL old_export_hash) + message(FATAL_ERROR "A node completion failure reached the later writer and replaced the owned export sentinel:\n${output}") + endif() +endif() + +if(TEST_MODE STREQUAL "flush" OR TEST_MODE STREQUAL "execute_and_flush" OR TEST_MODE STREQUAL "allocation") + if(TEST_MODE STREQUAL "allocation") + set(expected_code -272) + else() + set(expected_code -76402) + endif() + string(REGEX MATCH "Code: +${expected_code}[^\r\n]*C2-final-flush-error: cli-backing.h5:/Values" flush_line "${output}") + if(flush_line STREQUAL "") + message(FATAL_ERROR "Missing final checked error with exact backing context:\n${output}") + endif() + string(FIND "${output}" "C2-final-flush-warning" flush_warning) + if(flush_warning LESS 0) + message(FATAL_ERROR "The final flush warning was lost:\n${output}") + endif() + if(TEST_MODE STREQUAL "execute_and_flush") + string(FIND "${output}" "C2-original-execute-error" original_position) + string(FIND "${output}" "C2-final-flush-error" flush_position) + if(original_position LESS 0 OR original_position GREATER_EQUAL flush_position) + message(FATAL_ERROR "Original execute error did not precede the completion error:\n${output}") + endif() + endif() +else() + if(TEST_MODE STREQUAL "snapshot_allocation") + set(expected_code -272) + else() + set(expected_code -6071) + endif() + string(REGEX MATCHALL "Code: +${expected_code}([^0-9]|$)" snapshot_errors "${output}") + list(LENGTH snapshot_errors error_count) + if(TEST_MODE STREQUAL "root_snapshot") + set(expected_count 1) + else() + set(expected_count 2) + endif() + if(NOT error_count EQUAL expected_count) + message(FATAL_ERROR "Expected ${expected_count} snapshot diagnostics, got ${error_count}:\n${output}") + endif() + string(REGEX MATCHALL "Pipeline completion errors" root_headings "${output}") + list(LENGTH root_headings root_heading_count) + if(NOT root_heading_count EQUAL 1) + message(FATAL_ERROR "The root completion detail was missing or duplicated:\n${output}") + endif() +endif() +message(STATUS "Completion mode ${TEST_MODE} returned the expected pipeline error and diagnostic evidence.") diff --git a/src/nxrunner/test/PreferencesDefaultsProbe.cpp b/src/nxrunner/test/PreferencesDefaultsProbe.cpp new file mode 100644 index 0000000000..96bbe2a27f --- /dev/null +++ b/src/nxrunner/test/PreferencesDefaultsProbe.cpp @@ -0,0 +1,11 @@ +#include "simplnx/Core/Preferences.hpp" + +#include + +int main() +{ + nx::core::Preferences preferences; + std::cout << preferences.defaultValueAs(nx::core::Preferences::k_LargeDataSize_Key) << '\n'; + std::cout << preferences.defaultValueAs(nx::core::Preferences::k_LargeDataStructureSize_Key) << '\n'; + return 0; +} diff --git a/src/simplnx/Common/BoundedRead.hpp b/src/simplnx/Common/BoundedRead.hpp new file mode 100644 index 0000000000..9325496b2c --- /dev/null +++ b/src/simplnx/Common/BoundedRead.hpp @@ -0,0 +1,616 @@ +#pragma once + +#include "simplnx/Common/Extent.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nx::core::BoundedRead +{ +/** @brief Funds fixed diagnostic text and one terminal result record. */ +inline constexpr uint64 k_DiagnosticBytes = 4096; +/** @brief Limits the explicitly supported rank without importing HDF5 into stores. */ +inline constexpr usize k_MaxRank = 32; + +/** + * @enum Status + * @brief Separates completion, optional refusal and a failed operation. + */ +enum class Status +{ + Complete, ///< Every selected destination value is valid. + Unavailable, ///< No payload or destination transfer took place. + Failed ///< The caller discards the destination, which may contain a prefix. +}; + +/** + * @enum Code + * @brief Supplies stable bounded-read errors and refusal reasons. + */ +enum Code : int32 +{ + Insufficient = 6170, + Unsupported = 6171, + UnsupportedFilter = 6172, + PendingWrites = 6173, + UndefinedFill = 6174, + InvalidExtent = -6170, + MetadataFailure = -6171, + AllocationFailure = -6172, + ReadFailure = -6173, + DecodeFailure = -6174, + DecoderQuotaFailure = -6175, + MetadataChanged = -6176 +}; + +/** + * @brief Checks an unsigned product before storing it. + * @param first Supplies the first factor. + * @param second Supplies the second factor. + * @param result Receives the product on success. + * @return False without mutation when the product exceeds uint64. + */ +inline bool Multiply(uint64 first, uint64 second, uint64& result) noexcept +{ + if(second != 0 && first > std::numeric_limits::max() / second) + { + return false; + } + result = first * second; + return true; +} + +/** + * @brief Checks an unsigned sum before storing it. + * @param first Supplies the first term. + * @param second Supplies the second term. + * @param result Receives the sum on success. + * @return False without mutation when the sum exceeds uint64. + */ +inline bool Add(uint64 first, uint64 second, uint64& result) noexcept +{ + if(second > std::numeric_limits::max() - first) + { + return false; + } + result = first + second; + return true; +} + +/** + * @brief Reports reviewed standard-library terminal and counted-container representations. + * @return True only for source-reviewed compiler-library configurations. + * @note Native support tests must fail on an unreviewed configuration. This is not a platform opt-out. + * Apple's libc++ 180100 (Xcode MacOSX15.2.sdk) was reviewed for exact counted + * default-allocator vector allocation, allocation-free empty controls and moves, + * and bounded string reserve. Other libc++ 18 releases require their own review. + * Captured Apple libc++ 210106 headers (Xcode 26.6, macOS 26.5 SDK) + * satisfy these bounds with ABI 1, C++20 and ordinary new. + * Typed new and other libc++ 21 releases require separate source review. + * Apple libc++ 220106 headers (Xcode 27.0, build 27A5252f) satisfy these bounds with ABI 1, C++20 and ordinary new. + * Typed new and other libc++ 22 releases require separate source review. + * GCC 11.4 libstdc++ (__GLIBCXX__ == 20230528, CXX11 ABI) satisfies the same + * bounds as GCC 14. The review covers bits/stl_vector.h, bits/vector.tcc, + * bits/basic_string.h, bits/basic_string.tcc and expected-lite 0.8.0. + * + * GCC 13.3.0 satisfies these bounds with the CXX11 ABI. + * Its source review uses gcc.git commit b71f1de6e9cf7181a288c0f39f9b1ef6580cf5c8. + * GCC 13 admission follows the existing release-family policy and still requires native CI. + */ +inline constexpr bool DiagnosticRepresentationSupported() noexcept +{ +#if !defined(expected_lite_MAJOR) || expected_lite_MAJOR != 0 || expected_lite_MINOR != 8 || expected_lite_PATCH != 0 || nsel_USES_STD_EXPECTED + return false; +#elif defined(_MSVC_STL_VERSION) && defined(_MSVC_STL_UPDATE) + return _MSVC_STL_VERSION == 143 && _MSVC_STL_UPDATE == 202503L; +#elif defined(_GLIBCXX_RELEASE) + return (_GLIBCXX_RELEASE == 11 || _GLIBCXX_RELEASE == 13 || _GLIBCXX_RELEASE == 14) && _GLIBCXX_USE_CXX11_ABI == 1; +#elif defined(__APPLE__) && defined(__clang__) && defined(_LIBCPP_VERSION) && _LIBCPP_VERSION == 210106 && defined(_LIBCPP_ABI_VERSION) + return _LIBCPP_ABI_VERSION == 1 && __cplusplus == 202002L && !__has_feature(typed_cxx_new_delete); +#elif defined(__APPLE__) && defined(__clang__) && defined(_LIBCPP_VERSION) && _LIBCPP_VERSION == 220106 && defined(_LIBCPP_ABI_VERSION) + return _LIBCPP_ABI_VERSION == 1 && __cplusplus == 202002L && !__has_feature(typed_cxx_new_delete); +#elif defined(_LIBCPP_VERSION) + return _LIBCPP_VERSION == 180100 || (_LIBCPP_VERSION >= 190000 && _LIBCPP_VERSION < 200000); +#else + return false; +#endif +} + +/** + * @brief Bounds the fixed empty-vector proxy in the reviewed configuration. + * @return Two pointers for MSVC iterator debugging, otherwise zero. + * @note GCC 11.4/13.3/14, libc++ 180100/19.x and reviewed Apple libc++ 210106 and 220106 have no heap iterator proxy. + */ +inline constexpr uint64 EmptyVectorProxyBytes() noexcept +{ +#if defined(_MSVC_STL_VERSION) && defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL != 0 + return 2 * sizeof(void*); +#else + return 0; +#endif +} + +/** + * @brief Bounds caller-funded mandatory empty Result return controls. + * @return Two Result objects and warning-vector proxies for a named return and receiving object. + * @pre DiagnosticRepresentationSupported() is true for an accepted bound. + * @note Transparent prvalue forwarding adds no object in C++17. Nonempty diagnostics use their own D slice. + */ +inline constexpr uint64 EmptyResultControlBytes() noexcept +{ + return 2 * (sizeof(Result) + EmptyVectorProxyBytes()); +} + +/** + * @brief Bounds typed empty-invalid return controls without constructing a value payload. + * @tparam Value Specifies the checked API's inactive value representation. + * @return Two Result controls, empty vector/expected/unexpected temporaries, and their fixed proxies. + * @pre DiagnosticRepresentationSupported() is true for an accepted source bound. + * @note The direct empty-error construction has no Error elements or message text. + * Seven proxy slots cover two typed results, one standalone error vector, one + * unexpected carrier and one expected construction carrier. C++17 prvalue forwarding + * removes extra returned objects; the second Result slot covers a named caller move. + */ +template +inline constexpr uint64 EmptyInvalidResultControlBytes() noexcept +{ + using UnexpectedErrors = decltype(nonstd::make_unexpected(std::declval())); + return 2 * sizeof(Result) + sizeof(ErrorCollection) + sizeof(UnexpectedErrors) + sizeof(nonstd::expected) + 7 * EmptyVectorProxyBytes(); +} + +/** + * @brief Supplies the complete pre-service bound for one terminal diagnostic. + * @return Zero for an unreviewed representation; otherwise fixed controls, collection and text bytes. + */ +inline constexpr uint64 TerminalDiagnosticBoundBytes() noexcept; + +/** + * @brief Bounds a fresh counted default-allocator vector on reviewed STL releases. + * @param count Supplies elements for counted construction or fresh reserve. + * @param width Supplies bytes per non-owning, non-bool element. + * @param bytes Receives payload and a conservative debug iterator-proxy allowance. + * @return False for an unreviewed STL or overflow, without changing bytes. + * @note Native CI must exercise this guard; a toolchain upgrade requires source review. + * GCC 11.4/13.3/14 and libc++ 180100 counted construction and fresh reserve request exactly count elements. + * The reviewed Apple libc++ 210106 and 220106 configurations have the same allocation behavior. + * These implementations have no heap iterator proxy, so the 64-byte allowance is conservative. + */ +inline bool FreshVectorBytes(uint64 count, uint64 width, uint64& bytes) noexcept +{ + uint64 value = 0; + if(!DiagnosticRepresentationSupported() || width == 0 || !Multiply(count, width, value) || !Add(value, 64, value)) + { + return false; + } + bytes = value; + return true; +} + +#if SIMPLNX_BUILD_TESTS +/** + * @brief Reports a terminal allocation request or public retained capacities through the shared codec observer. + * @param file Borrows an optional backend identity; null selects an unobserved resident diagnostic. + * @param dataset Borrows the dataset identity. + * @param requested Selects request bytes instead of retained capacities. + * @param first Supplies requested bytes or retained record-vector bytes. + * @param second Selects request kind (zero vector, one message), or retained message bytes. + */ +SIMPLNX_EXPORT void ObserveTerminalForTesting(const std::filesystem::path* file, std::string_view dataset, bool requested, uint64 first, uint64 second) noexcept; +#endif + +/** + * @struct Diagnostic + * @brief Owns one bounded message until payload scratch has been destroyed. + */ +struct Diagnostic +{ + std::array text{}; + usize length = 0; + int32 code = 0; + bool truncated = false; +#if SIMPLNX_BUILD_TESTS + const std::filesystem::path* observedFile = nullptr; + std::string_view observedDataset; +#endif + + /** + * @brief Appends text without allocating. + * @param value Supplies borrowed diagnostic text. + */ + void append(std::string_view value) noexcept + { + constexpr auto hex = std::to_array("0123456789ABCDEF"); + usize index = 0; + while(index < value.size()) + { + const auto first = static_cast(value[index]); + usize count = first < 0x80 ? 1 : first >= 0xC2 && first <= 0xDF ? 2 : first >= 0xE0 && first <= 0xEF ? 3 : first >= 0xF0 && first <= 0xF4 ? 4 : 0; + bool valid = count != 0 && count <= value.size() - index; + for(usize offset = 1; valid && offset < count; ++offset) + { + const auto next = static_cast(value[index + offset]); + valid = (next & 0xC0U) == 0x80U; + } + if(valid && count > 1) + { + const auto second = static_cast(value[index + 1]); + valid = !((first == 0xE0 && second < 0xA0) || (first == 0xED && second >= 0xA0) || (first == 0xF0 && second < 0x90) || (first == 0xF4 && second >= 0x90)); + } + std::array escaped{}; + const char* bytes = value.data() + index; + usize consumed = count; + if(!valid || first < 0x20 || first == 0x7F) + { + escaped = {'\\', 'x', hex[first >> 4], hex[first & 0x0F]}; + bytes = escaped.data(); + count = escaped.size(); + consumed = 1; + } + if(count > text.size() - 1 - length) + { + truncated = true; + return; + } + std::memcpy(text.data() + length, bytes, count); + length += count; + text[length] = '\0'; + index += consumed; + } + } + /** + * @brief Appends a checked unsigned value without formatting allocation. + * @param value Supplies the numeric context. + */ + void number(uint64 value) noexcept + { + std::array buffer{}; + const auto converted = std::to_chars(buffer.data(), buffer.data() + buffer.size(), value); + if(converted.ec == std::errc{}) + { + append({buffer.data(), static_cast(converted.ptr - buffer.data())}); + } + } + /** + * @brief Replaces diagnostic state with one terminal reason. + * @param value Supplies the stable error or warning code. + * @param message Supplies its fixed context prefix. + * @return Failed for a negative code, otherwise Unavailable. + */ + Status set(int32 value, std::string_view message) noexcept + { + code = value; + length = 0; + text[0] = '\0'; + truncated = false; + append(message); + return code < 0 ? Status::Failed : Status::Unavailable; + } + /** + * @brief Sets one numeric required/available diagnostic without allocation. + * @param value Supplies the stable error or warning code. + * @param message Supplies the fixed diagnostic prefix. + * @param required Supplies the actual required count or bytes. + * @param available Supplies the admitted or actual available count or bytes. + * @return The status selected by value. + */ + Status counts(int32 value, std::string_view message, uint64 required, uint64 available) noexcept + { + const auto status = set(value, message); + append("; required="); + number(required); + append(", available="); + number(available); + return status; + } + + /** + * @brief Adds a bounded, valid UTF-8 path prefix with explicit native-unit escaping. + * @param path Borrows the existing native representation. + * @param dataset Borrows the dataset name; invalid UTF-8/control bytes use hexadecimal escapes. + * @note Diagnostic encoding never selects the filename passed to the opening adapter. + */ + void context(const std::filesystem::path& path, std::string_view dataset) noexcept + { +#if SIMPLNX_BUILD_TESTS + observedFile = &path; + observedDataset = dataset; +#endif + const auto& native = path.native(); + usize count = std::min(native.size(), 120); +#ifdef _WIN32 + append(" [native UTF-16 path; malformed units escaped: "); + constexpr auto hex = std::to_array("0123456789ABCDEF"); + for(usize index = 0; index < count; ++index) + { + uint32 value = static_cast(native[index]); + if(value >= 0xD800 && value <= 0xDBFF && index + 1 < native.size()) + { + const uint32 next = static_cast(native[index + 1]); + if(next >= 0xDC00 && next <= 0xDFFF) + { + if(index + 1 >= count) + { + count = index; + break; + } + value = 0x10000 + ((value - 0xD800) << 10) + (next - 0xDC00); + ++index; + } + } + if(value >= 0xD800 && value <= 0xDFFF) + { + const std::array escaped{'\\', 'u', hex[(value >> 12) & 15], hex[(value >> 8) & 15], hex[(value >> 4) & 15], hex[value & 15]}; + append({escaped.data(), escaped.size()}); + continue; + } + std::array encoded{}; + usize bytes = 0; + if(value < 0x80) + { + encoded[bytes++] = static_cast(value); + } + else if(value < 0x800) + { + encoded[bytes++] = static_cast(0xC0U | (value >> 6)); + encoded[bytes++] = static_cast(0x80U | (value & 0x3FU)); + } + else if(value < 0x10000) + { + encoded[bytes++] = static_cast(0xE0U | (value >> 12)); + encoded[bytes++] = static_cast(0x80U | ((value >> 6) & 0x3FU)); + encoded[bytes++] = static_cast(0x80U | (value & 0x3FU)); + } + else + { + encoded[bytes++] = static_cast(0xF0U | (value >> 18)); + encoded[bytes++] = static_cast(0x80U | ((value >> 12) & 0x3FU)); + encoded[bytes++] = static_cast(0x80U | ((value >> 6) & 0x3FU)); + encoded[bytes++] = static_cast(0x80U | (value & 0x3FU)); + } + append({encoded.data(), bytes}); + } +#else + append(" [native path; invalid UTF-8/control bytes escaped: "); + if(count < native.size()) + { + while(count != 0 && (static_cast(native[count]) & 0xC0U) == 0x80U) + { + --count; + } + } + append({native.data(), count}); +#endif + if(count != native.size()) + { + append("..."); + truncated = true; + } + append("; dataset (invalid/control bytes escaped): "); + append(dataset); + append("]"); + } + /** + * @brief Constructs the single active terminal collection after scratch destruction. + * @param status Supplies the operation outcome. + * @return Completion, reason-coded refusal, or one error without initializer-list message copies. + * @throws std::bad_alloc If even the admitted terminal record cannot be allocated. + */ + Result finish(Status status) + { + // The unsupported-build exception permits only mandatory empty return bookkeeping. + // It never admits collection elements or diagnostic text before this proof. + if(!DiagnosticRepresentationSupported() || TerminalDiagnosticBoundBytes() > k_DiagnosticBytes) + { + return {false}; + } + if(status == Status::Complete) + { + return {true}; + } + if(truncated) + { + constexpr std::string_view marker = " [truncated]"; + usize start = std::min(length, text.size() - 1 - marker.size()); + while(start != 0 && start < length && (static_cast(text[start]) & 0xC0U) == 0x80U) + { + --start; + } + std::memcpy(text.data() + start, marker.data(), marker.size()); + length = start + marker.size(); + text[length] = '\0'; + } + if(status == Status::Unavailable) + { + Result result{false}; +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, true, sizeof(Warning), 0); +#endif + result.warnings().reserve(1); +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, false, result.warnings().capacity() * sizeof(Warning), 0); +#endif + result.warnings().emplace_back(); + auto& warning = result.warnings().back(); + warning.code = code; +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, true, text.size() - 1, 1); +#endif + warning.message.reserve(text.size() - 1); +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, false, result.warnings().capacity() * sizeof(Warning), warning.message.capacity() + 1); +#endif + warning.message.assign(text.data(), length); + return result; + } + std::vector errors; +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, true, sizeof(Error), 0); +#endif + errors.reserve(1); +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, false, errors.capacity() * sizeof(Error), 0); +#endif + errors.emplace_back(); + errors.back().code = code; +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, true, text.size() - 1, 1); +#endif + errors.back().message.reserve(text.size() - 1); +#if SIMPLNX_BUILD_TESTS + ObserveTerminalForTesting(observedFile, observedDataset, false, errors.capacity() * sizeof(Error), errors.back().message.capacity() + 1); +#endif + errors.back().message.assign(text.data(), length); + Result result{false}; + result.m_Expected = nonstd::make_unexpected(std::move(errors)); + return result; + } +}; + +/** + * @brief Calculates the reviewed terminal lifetime envelope before any reserve. + * @return A proven upper bound no larger than D on a supported library, or zero when unreviewed. + * + * MSVC counted vector reserve is exact. Its string growth rounds the requested + * capacity and can use geometric growth. The fresh 511-character request fits + * 2*(511+1)+64 bytes, plus a separately bounded proxy. GCC 11.4/13.3/14 counted reserve is + * exact and fresh string creation fits that same bound. GCC 11.4/13.3 have SSO capacity 15. + * On growth, _M_create doubles capacity only when the request is below twice + * the old capacity. Fresh reserve(511) requests exactly 512 bytes. + * Subsequent assign of at most 511 characters does not grow it. + * libc++ 180100 and 19.x round fresh string capacity to their allocation alignment + * and also fit that bound. In Apple's libc++ 180100, reserve(511) requests 512 bytes + * including the terminator; subsequent assign does not grow it. + * The reviewed Apple libc++ 210106 and 220106 configurations also request 512 bytes for reserve(511). + * Assigning at most 511 characters does not grow its message buffer. + * These GNU and libc++ implementations + * have no empty-vector/string heap proxy. Default-allocator moves/swap transfer + * the existing payload without allocation. + * + * expected-lite 0.8 assignment builds an expected temporary, then swaps through + * one local error vector. Together with the local ErrorCollection, unexpected + * temporary and two Result representations, eight vector proxy slots suffice. + * The payload moves between these controls. Named return optimization is not required. + */ +inline constexpr uint64 TerminalDiagnosticBoundBytes() noexcept +{ + if(!DiagnosticRepresentationSupported()) + { + return 0; + } + using UnexpectedErrors = decltype(nonstd::make_unexpected(std::declval())); + constexpr uint64 resultControls = 2 * sizeof(Result); + constexpr uint64 errorMoveControls = 2 * sizeof(ErrorCollection) + sizeof(UnexpectedErrors) + sizeof(nonstd::expected); + constexpr uint64 proxies = 8 * EmptyVectorProxyBytes(); + // Numeric conversion, native/UTF-8 escaping and bounded string-view arguments. + constexpr uint64 formattingControls = + sizeof(std::to_chars_result) + sizeof(std::array) + 2 * sizeof(std::array) + sizeof(std::array) + 8 * sizeof(std::string_view) + 4 * sizeof(usize); + constexpr uint64 recordPayload = std::max(sizeof(Error), sizeof(Warning)) + 64; + constexpr uint64 messagePayloadAndProxy = 2 * 512 + 64 + 64; + return sizeof(Diagnostic) + resultControls + errorMoveControls + proxies + formattingControls + recordPayload + messagePayloadAndProxy; +} +static_assert(!DiagnosticRepresentationSupported() || TerminalDiagnosticBoundBytes() <= k_DiagnosticBytes); + +/** + * @brief Measures the active terminal collection's public retained capacities. + * @param result Borrows the terminal result through synchronous observation. + * @return Record-vector payload plus message capacities and terminators. + * @note This is an owned-capacity witness, not allocator bookkeeping or process RSS. + */ +inline uint64 ResultCapacityBytes(const Result& result) noexcept +{ + uint64 bytes = result.warnings().capacity() * sizeof(Warning); + for(const auto& warning : result.warnings()) + { + bytes += warning.message.capacity() + 1; + } + if(result.invalid()) + { + bytes += result.errors().capacity() * sizeof(Error); + for(const auto& error : result.errors()) + { + bytes += error.message.capacity() + 1; + } + } + return bytes; +} + +/** + * @brief Validates tuple selection arithmetic without allocating or calling a store reader. + * @tparam Shape Specifies the existing shape container. + * @param extent Supplies the selected tuple coordinates. + * @param shape Supplies positive source dimensions. + * @param components Supplies the checked component count. + * @param destinationValues Supplies the exact destination element count. + * @param selectedTuples Receives the selected tuple count. + * @param diagnostic Receives a fixed validation failure. + * @param elementBytes Supplies scalar width for addressable byte-count checks. + * @return Complete when ranks, bounds, products and destination count agree. + */ +template +Status ValidateExtent(const Extent& extent, const Shape& shape, uint64 components, uint64 destinationValues, uint64& selectedTuples, Diagnostic& diagnostic, uint64 elementBytes = 1) noexcept +{ + const usize rank = shape.size(); + if(rank > k_MaxRank) + { + return diagnostic.counts(Unsupported, "Bounded read rank exceeds the supported maximum.", rank, k_MaxRank); + } + if(extent.min.size() != rank || extent.max.size() != rank || extent.stride.size() != rank) + { + diagnostic.counts(InvalidExtent, "Bounded read extent minimum-vector rank does not match the source.", rank, extent.min.size()); + diagnostic.append(", maximum-vector rank="); + diagnostic.number(extent.max.size()); + diagnostic.append(", stride-vector rank="); + diagnostic.number(extent.stride.size()); + return Status::Failed; + } + uint64 sourceTuples = 1; + selectedTuples = 1; + for(usize dimension = 0; dimension < rank; ++dimension) + { + const uint64 bound = static_cast(shape[dimension]); + if(bound == 0 || extent.stride[dimension] == 0 || extent.min[dimension] > extent.max[dimension] || extent.max[dimension] >= bound) + { + diagnostic.set(InvalidExtent, "Bounded read has invalid bounds/stride at dimension "); + diagnostic.number(dimension); + diagnostic.append("; minimum="); + diagnostic.number(extent.min[dimension]); + diagnostic.append(", maximum="); + diagnostic.number(extent.max[dimension]); + diagnostic.append(", bound="); + diagnostic.number(bound); + diagnostic.append(", stride="); + diagnostic.number(extent.stride[dimension]); + return Status::Failed; + } + const uint64 count = (extent.max[dimension] - extent.min[dimension]) / extent.stride[dimension] + 1; + if(!Multiply(sourceTuples, bound, sourceTuples) || !Multiply(selectedTuples, count, selectedTuples)) + { + return diagnostic.set(InvalidExtent, "Bounded read shape or selected count overflows uint64."); + } + } + uint64 sourceValues = 0; + uint64 selectedValues = 0; + if(elementBytes == 0 || !Multiply(sourceTuples, components, sourceValues) || sourceValues > std::numeric_limits::max() / elementBytes || + !Multiply(selectedTuples, components, selectedValues) || selectedValues > std::numeric_limits::max() / elementBytes || selectedValues != destinationValues) + { + diagnostic.set(InvalidExtent, "Bounded read value count is invalid; destination="); + diagnostic.number(destinationValues); + diagnostic.append(", components="); + diagnostic.number(components); + diagnostic.append(", tuples="); + diagnostic.number(selectedTuples); + return Status::Failed; + } + return Status::Complete; +} +} // namespace nx::core::BoundedRead diff --git a/src/simplnx/Common/TypesUtility.hpp b/src/simplnx/Common/TypesUtility.hpp index d1947881a5..bcdcbd5b9c 100644 --- a/src/simplnx/Common/TypesUtility.hpp +++ b/src/simplnx/Common/TypesUtility.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -18,9 +19,9 @@ namespace nx::core { /** - * @brief Returns the templated value for the byte pattern 0xAB based on the byte count of the template parameter - * @tparam T - * @return + * @brief Returns a diagnostic value with a valid representation for the value type. + * @tparam T Numeric value type. + * @return Canonical true for bool; the repeated 0xAB byte pattern for other numeric types. */ template constexpr T GetMudflap() noexcept @@ -31,19 +32,23 @@ constexpr T GetMudflap() noexcept namespace bs = nx::core; #endif - if constexpr(sizeof(T) == 1) + if constexpr(std::is_same_v) + { + return true; + } + else if constexpr(sizeof(T) == 1) { return bs::bit_cast(static_cast(0xAB)); } - if constexpr(sizeof(T) == 2) + else if constexpr(sizeof(T) == 2) { return bs::bit_cast(static_cast(0xABAB)); } - if constexpr(sizeof(T) == 4) + else if constexpr(sizeof(T) == 4) { return bs::bit_cast(static_cast(0xABABABAB)); } - if constexpr(sizeof(T) == 8) + else if constexpr(sizeof(T) == 8) { return bs::bit_cast(static_cast(0xABABABABABABABAB)); } diff --git a/src/simplnx/Core/Application.cpp b/src/simplnx/Core/Application.cpp index 6ad71bac45..7e38949af7 100644 --- a/src/simplnx/Core/Application.cpp +++ b/src/simplnx/Core/Application.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #if defined(__linux__) @@ -206,6 +207,7 @@ Result<> Application::loadPreferences(const std::filesystem::path& filepath) m_Preferences = std::make_unique(); } auto result = m_Preferences->loadFromFile(filepath); + m_PendingPreferenceLoadWarnings.insert(m_PendingPreferenceLoadWarnings.end(), result.warnings().begin(), result.warnings().end()); if(result.invalid()) { if(!result.errors().empty() && result.errors()[0].code == Preferences::k_FileDoesNotExist_Code) @@ -217,6 +219,12 @@ Result<> Application::loadPreferences(const std::filesystem::path& filepath) } return result; } + +WarningCollection Application::takePreferenceLoadWarnings() +{ + return std::exchange(m_PendingPreferenceLoadWarnings, WarningCollection{}); +} + Result<> Application::savePreferences() { if(m_Preferences == nullptr) diff --git a/src/simplnx/Core/Application.hpp b/src/simplnx/Core/Application.hpp index 5780b91285..337ae3f761 100644 --- a/src/simplnx/Core/Application.hpp +++ b/src/simplnx/Core/Application.hpp @@ -129,6 +129,12 @@ class SIMPLNX_EXPORT Application */ Result<> loadPreferences(const std::filesystem::path& filepath); + /** + * @brief Takes pending preference-load warnings once. + * @return Warnings from all loads since the previous take, in load order. + */ + WarningCollection takePreferenceLoadWarnings(); + /** * @brief Returns no JSON pipeline builder. * @return nullptr. @@ -202,7 +208,9 @@ class SIMPLNX_EXPORT Application /** * @brief Initializes preferences, executable path, and type mappings. - * @return Preference-load warnings or an executable-path error. + * @return Preference-load or executable-path errors. + * + * Preference-load warnings remain pending for a later consumer. */ Result<> initialize(); @@ -228,5 +236,6 @@ class SIMPLNX_EXPORT Application std::shared_ptr m_DataIOCollection; name_type_map m_NamedTypesMap; std::unique_ptr m_Preferences = nullptr; + WarningCollection m_PendingPreferenceLoadWarnings; }; } // namespace nx::core diff --git a/src/simplnx/Core/Preferences.cpp b/src/simplnx/Core/Preferences.cpp index dac64cbd72..f66fcedd10 100644 --- a/src/simplnx/Core/Preferences.cpp +++ b/src/simplnx/Core/Preferences.cpp @@ -1,4 +1,8 @@ #include "Preferences.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include +#include #include "simplnx/Common/SimplnxConfig.hpp" #include "simplnx/Core/Application.hpp" @@ -6,7 +10,18 @@ #include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/MemoryUtilities.hpp" +#include + +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #ifdef _WIN32 #include @@ -29,11 +44,289 @@ constexpr StringLiteral k_DefaultFileName = "preferences.json"; constexpr int64 k_ReducedDataStructureSize = 3221225472; // 3 GB constexpr bool k_AutoRangeComputationDefault = false; +constexpr int32 k_InvalidRootCode = -589; +constexpr int32 k_InvalidOocSizeCode = -590; +constexpr int32 k_RepairedOocSizeWarning = -591; +constexpr int32 k_RepairedOocSizeGBWarning = -592; + +// Correctly rounded division gives the double nearest to 17179869183.99, which is also the parsed JSON value. +constexpr double k_MaxOocSizeGB = static_cast(Preferences::k_MaxOocSizeHundredths) / 100.0; + constexpr StringLiteral k_FailedToCreateDirectory_Message = "Failed to create the parent directory when saving Preferences. Check that the path is valid and writable."; constexpr StringLiteral k_FileDoesNotExist_Message = "Preferences file does not exist"; constexpr StringLiteral k_FileCouldNotOpen_Message = "Could not open Preferences file"; constexpr StringLiteral k_JsonParseError_Message = "Parsing the JSON Preferences file failed."; +std::string_view oocSizeLabel(const std::string& name) +{ + if(name == Preferences::k_LargeDataSize_Key) + { + return "Large Data Size"; + } + if(name == Preferences::k_LargeDataStructureSize_Key) + { + return "Large Data Structure Size"; + } + return {}; +} + +/** + * @struct OocSizeKeys + * @brief Names the in-memory byte key and the file GB key of one out-of-core size threshold. + */ +struct OocSizeKeys +{ + std::string bytesKey; + std::string gbKey; +}; + +/** + * @brief Returns the byte and GB keys of both out-of-core size thresholds, Large Data Size first. + * @return Key pairs in the order that load warnings use. + */ +std::array oocSizeKeys() +{ + return {OocSizeKeys{std::string(Preferences::k_LargeDataSize_Key), std::string(Preferences::k_LargeDataSizeGB_Key)}, + OocSizeKeys{std::string(Preferences::k_LargeDataStructureSize_Key), std::string(Preferences::k_LargeDataStructureSizeGB_Key)}}; +} + +/** + * @brief Formats hundredths of a GB as exact text with two decimals. + * @param hundredths Value in hundredths of a GB. + * @return Text such as "1.00" or "17179869183.99". + */ +std::string formatHundredthsAsGB(uint64 hundredths) +{ + return fmt::format("{}.{:02}", hundredths / 100, hundredths % 100); +} + +/** + * @brief Converts a saved GB value to hundredths of a GB. + * @param value Saved JSON value. + * @return Hundredths of a GB, or std::nullopt when the value is not a finite JSON number from 0 to k_MaxOocSizeGB. + * + * A value above 0 that rounds to 0 hundredths becomes 1 (0.01 GB). + */ +std::optional savedGBToHundredths(const nlohmann::json& value) +{ + if(!value.is_number()) + { + return std::nullopt; + } + const double gigabytes = value.get(); + // NaN fails both comparisons. The range check also rejects infinity before the integer conversion. + if(!(gigabytes >= 0.0 && gigabytes <= k_MaxOocSizeGB)) + { + return std::nullopt; + } + auto hundredths = static_cast(std::llround(gigabytes * 100.0)); + if(gigabytes > 0.0 && hundredths == 0) + { + hundredths = 1; + } + return hundredths; +} + +std::string_view jsonTypeName(const nlohmann::json& value) +{ + if(value.is_number_unsigned()) + { + return "number_unsigned"; + } + if(value.is_number_integer()) + { + return "number_integer"; + } + if(value.is_number_float()) + { + return "number_float"; + } + return value.type_name(); +} + +bool canRenderExact(const nlohmann::json& value, std::size_t& remaining, int depth) +{ + if(depth > 3 || remaining < 32) + { + return false; + } + if(value.is_string()) + { + const auto& contents = value.get_ref(); + if(contents.size() > (remaining - 2) / 6) + { + return false; + } + remaining -= 2 + 6 * contents.size(); + return true; + } + if(value.is_array() || value.is_object()) + { + if(value.size() > 8 || remaining < 2 + 2 * value.size()) + { + return false; + } + remaining -= 2 + 2 * value.size(); + for(auto iter = value.begin(); iter != value.end(); ++iter) + { + if(value.is_object()) + { + if(iter.key().size() > remaining / 6) + { + return false; + } + remaining -= 6 * iter.key().size(); + } + if(!canRenderExact(iter.value(), remaining, depth + 1)) + { + return false; + } + } + return true; + } + if(value.is_binary()) + { + return false; + } + if(value.is_number_float() && !std::isfinite(value.get())) + { + return false; + } + remaining -= 32; + return true; +} + +std::string boundedPreview(const nlohmann::json& value, int depth) +{ + if(value.is_number_float()) + { + const double number = value.get(); + if(std::isnan(number)) + { + return "NaN"; + } + if(std::isinf(number)) + { + return std::signbit(number) ? "-Infinity" : "+Infinity"; + } + } + if(value.is_string()) + { + const auto& contents = value.get_ref(); + std::string preview = nlohmann::json(contents.substr(0, 24)).dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + if(contents.size() > 24) + { + preview += "..."; + } + return preview; + } + if(value.is_array() || value.is_object()) + { + if(depth >= 2) + { + return fmt::format("[{} with {} entries]", jsonTypeName(value), value.size()); + } + std::string preview = value.is_object() ? "{" : "["; + std::size_t shown = 0; + for(auto iter = value.begin(); iter != value.end() && shown < 2; ++iter, ++shown) + { + if(shown != 0) + { + preview += ","; + } + if(value.is_object()) + { + const auto& key = iter.key(); + preview += nlohmann::json(key.substr(0, 24)).dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + if(key.size() > 24) + { + preview += "..."; + } + preview += ":"; + } + preview += boundedPreview(iter.value(), depth + 1); + } + if(value.size() > shown) + { + preview += ",..."; + } + preview += value.is_object() ? "}" : "]"; + if(preview.size() > 256) + { + preview.resize(253); + preview += "..."; + } + return preview; + } + if(value.is_binary()) + { + return "[binary value]"; + } + return value.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); +} + +std::string boundedJsonValue(const nlohmann::json& value) +{ + std::size_t remaining = 256; + if(canRenderExact(value, remaining, 0)) + { + return value.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + } + if(value.is_string()) + { + const auto& contents = value.get_ref(); + return nlohmann::json(contents.substr(0, 64)).dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "..."; + } + if(value.is_array() || value.is_object()) + { + return fmt::format("{} with {} entries; preview {} [truncated]", jsonTypeName(value), value.size(), boundedPreview(value, 0)); + } + return boundedPreview(value, 0); +} + +/** + * @brief Reads one saved out-of-core size threshold and converts it to bytes on the 0.01-GB grid. + * @param document Parsed preferences file. It contains keys.gbKey, keys.bytesKey, or both. + * @param keys Byte and GB keys of the threshold. + * @param defaultBytes Default threshold in bytes, which the warning names. + * @param warnings Receives one repair warning when the selected value is invalid. + * @return Threshold in bytes, or std::nullopt when the selected value is invalid. + * + * The GB key takes precedence, and the function does not validate a byte key next to it. + * A legacy byte value that ValidateOocSizeValue() accepts moves to the nearest grid value. + */ +std::optional readSavedOocSize(const nlohmann::json& document, const OocSizeKeys& keys, uint64 defaultBytes, std::vector& warnings) +{ + const std::string_view label = oocSizeLabel(keys.bytesKey); + if(document.contains(keys.gbKey)) + { + const nlohmann::json& saved = document.at(keys.gbKey); + const std::optional hundredths = savedGBToHundredths(saved); + if(hundredths.has_value()) + { + return Preferences::OocSizeHundredthsToBytes(*hundredths); + } + warnings.push_back(Warning{.code = k_RepairedOocSizeGBWarning, + .message = fmt::format("Preferences: saved {} ('{}') value {} (JSON type {}) is invalid. Allowed domain: 0 to {} GB. Using the default: {} GB. Review this " + "value in Preferences > Out of Core.", + label, keys.gbKey, boundedJsonValue(saved), jsonTypeName(saved), formatHundredthsAsGB(Preferences::k_MaxOocSizeHundredths), + formatHundredthsAsGB(Preferences::OocSizeBytesToHundredths(defaultBytes)))}); + return std::nullopt; + } + + const nlohmann::json& saved = document.at(keys.bytesKey); + const auto validation = Preferences::ValidateOocSizeValue(keys.bytesKey, saved); + if(validation.valid()) + { + return Preferences::OocSizeHundredthsToBytes(Preferences::OocSizeBytesToHundredths(validation.value())); + } + warnings.push_back(Warning{.code = k_RepairedOocSizeWarning, + .message = fmt::format("Preferences: saved {} ('{}') value {} (JSON type {}) is invalid. Allowed domain: 0 to {} bytes. Using the default: {} bytes. Review this value in " + "Preferences > Out of Core.", + label, keys.bytesKey, boundedJsonValue(saved), jsonTypeName(saved), std::numeric_limits::max(), defaultBytes)}); + return std::nullopt; +} + /** * @brief Returns the current user's home directory. * @return Home directory from the platform environment or account database. @@ -64,6 +357,29 @@ std::filesystem::path Preferences::DefaultFilePath(const std::string& applicatio #endif } +Result Preferences::ValidateOocSizeValue(const std::string& name, const nlohmann::json& value) +{ + const std::string_view label = oocSizeLabel(name); + if(label.empty()) + { + return MakeErrorResult(k_InvalidOocSizeCode, fmt::format("Preferences size key '{}' is not supported for value {} (JSON type {}).", name, boundedJsonValue(value), jsonTypeName(value))); + } + if(value.is_number_unsigned()) + { + return {value.get()}; + } + if(value.is_number_integer()) + { + const int64 signedValue = value.get(); + if(signedValue >= 0) + { + return {static_cast(signedValue)}; + } + } + return MakeErrorResult(k_InvalidOocSizeCode, fmt::format("Preferences {} ('{}') value {} (JSON type {}) is invalid. Allowed domain: 0 to {} bytes.", label, name, boundedJsonValue(value), + jsonTypeName(value), std::numeric_limits::max())); +} + Preferences::Preferences() { setDefaultValues(); @@ -162,9 +478,17 @@ nlohmann::json Preferences::defaultValue(const std::string& name) const void Preferences::setValue(const std::string& name, const nlohmann::json& value) { + if(name == k_LargeDataSize_Key || name == k_LargeDataStructureSize_Key) + { + const auto validation = ValidateOocSizeValue(name, value); + if(validation.invalid()) + { + throw std::invalid_argument(validation.errors().front().message); + } + } m_Values[name] = value; - // The single-array threshold determines the whole-data-structure default. + // The default whole-data-structure threshold uses the default array threshold. if(name == k_LargeDataSize_Key) { updateMemoryDefaults(); @@ -212,7 +536,20 @@ Result<> Preferences::saveToFile(const std::filesystem::path& filepath) const return MakeErrorResult(Preferences::k_FileCouldNotOpen_Code, k_FileCouldNotOpen_Message); } - fileStream << m_Values; + // The file stores the out-of-core size thresholds in GB. Memory keeps them in bytes. + nlohmann::json document = m_Values; + for(const auto& keys : oocSizeKeys()) + { + if(!document.contains(keys.bytesKey)) + { + continue; + } + const uint64 bytes = document.at(keys.bytesKey).get(); + document.erase(keys.bytesKey); + document[keys.gbKey] = static_cast(OocSizeBytesToHundredths(bytes)) / 100.0; + } + + fileStream << document; return {}; } @@ -235,7 +572,30 @@ Result<> Preferences::loadFromFile(const std::filesystem::path& filepath) return MakeErrorResult(Preferences::k_JsonParseError_Code, k_JsonParseError_Message); } - m_Values = parsedResult; + if(!parsedResult.is_object()) + { + return MakeErrorResult(k_InvalidRootCode, fmt::format("Preferences file '{}' has a JSON {} root. Expected a JSON object.", filepath.string(), jsonTypeName(parsedResult))); + } + + Result<> result; + for(const auto& keys : oocSizeKeys()) + { + if(!parsedResult.contains(keys.gbKey) && !parsedResult.contains(keys.bytesKey)) + { + continue; + } + // Validate the selected representation before the keys are removed. An invalid value stores + // nothing, so the default applies. + const std::optional gridBytes = readSavedOocSize(parsedResult, keys, defaultValueAs(keys.bytesKey), result.warnings()); + parsedResult.erase(keys.gbKey); + parsedResult.erase(keys.bytesKey); + if(gridBytes.has_value()) + { + parsedResult[keys.bytesKey] = *gridBytes; + } + } + + m_Values = std::move(parsedResult); // Preserve a canonical value when both keys exist. Remove the legacy key so // later saves retain only the cache-specific preference. @@ -257,7 +617,7 @@ Result<> Preferences::loadFromFile(const std::filesystem::path& filepath) } updateMemoryDefaults(); - return {}; + return result; } bool Preferences::useOocData() const @@ -322,7 +682,8 @@ void Preferences::updateMemoryDefaults() targetValue = totalMemory / 2; } - m_DefaultValues[k_LargeDataStructureSize_Key] = targetValue; + // A default on the 0.01-GB grid survives the GB file format and the Preferences dialog without a change. + m_DefaultValues[k_LargeDataStructureSize_Key] = OocSizeHundredthsToBytes(OocSizeBytesToHundredths(targetValue)); } uint64 Preferences::largeDataStructureSize() const diff --git a/src/simplnx/Core/Preferences.hpp b/src/simplnx/Core/Preferences.hpp index d9e461b804..2ee4a4b37f 100644 --- a/src/simplnx/Core/Preferences.hpp +++ b/src/simplnx/Core/Preferences.hpp @@ -56,6 +56,20 @@ class SIMPLNX_EXPORT Preferences static inline constexpr StringLiteral k_LargeDataStructureSize_Key = "large_datastructure_size"; + // The preferences file stores Large Data Size under this key, in GB with two decimals. + // Memory keeps the value in bytes under k_LargeDataSize_Key. + static constexpr StringLiteral k_LargeDataSizeGB_Key = "large_data_size_gb"; + + // The preferences file stores Large Data Structure Size under this key, in GB with two decimals. + // Memory keeps the value in bytes under k_LargeDataStructureSize_Key. + static constexpr StringLiteral k_LargeDataStructureSizeGB_Key = "large_datastructure_size_gb"; + + // One GB of an out-of-core size threshold is 1024^3 bytes. + static constexpr uint64 k_BytesPerGiB = 1073741824; + + // The largest threshold in hundredths of a GB (17179869183.99 GB) whose byte count fits in uint64. + static constexpr uint64 k_MaxOocSizeHundredths = 1717986918399; + // Migration reads this legacy key. The core does not write it. static inline constexpr StringLiteral k_ForceOocData_Key = "force_ooc_data"; @@ -81,6 +95,57 @@ class SIMPLNX_EXPORT Preferences */ static std::filesystem::path DefaultFilePath(const std::string& applicationName); + /** + * @brief Validates an out-of-core size threshold without changing preferences. + * @param name Large Data Size or Large Data Structure Size preference key. + * @param value Input JSON. + * + * @return Exact nonnegative integer in bytes, or a contextual error. + */ + static Result ValidateOocSizeValue(const std::string& name, const nlohmann::json& value); + + /** + * @brief Converts an out-of-core size threshold from hundredths of a GB to bytes. + * @param hundredths Threshold in hundredths of a GB (1 GB = 1024^3 bytes). + * @return Nearest byte count, with a half byte rounded up. + * + * The preferences file and the Preferences dialog store a threshold as a multiple of 0.01 GB. + * File load, file save and the dialog use this function and OocSizeBytesToHundredths() so that + * they apply one rounding rule. A value above k_MaxOocSizeHundredths converts as k_MaxOocSizeHundredths. + * The calculation does not overflow. + */ + static constexpr uint64 OocSizeHundredthsToBytes(uint64 hundredths) noexcept + { + const uint64 bounded = hundredths < k_MaxOocSizeHundredths ? hundredths : k_MaxOocSizeHundredths; + return (bounded / 100) * k_BytesPerGiB + ((bounded % 100) * k_BytesPerGiB + 50) / 100; + } + + /** + * @brief Converts an out-of-core size threshold from bytes to hundredths of a GB. + * @param bytes Threshold in bytes. + * @return Nearest number of hundredths of a GB (1 GB = 1024^3 bytes), with a half hundredth rounded up. + * + * The result is at most k_MaxOocSizeHundredths. A nonzero byte count that rounds to zero + * becomes 1 (0.01 GB), so a small positive threshold never becomes zero. For every value h + * from 0 to k_MaxOocSizeHundredths, OocSizeBytesToHundredths(OocSizeHundredthsToBytes(h)) is h. + * The calculation does not overflow. + */ + static constexpr uint64 OocSizeBytesToHundredths(uint64 bytes) noexcept + { + const uint64 wholeGiB = bytes / k_BytesPerGiB; + const uint64 remainderBytes = bytes % k_BytesPerGiB; + uint64 hundredths = wholeGiB * 100 + (remainderBytes * 100 + k_BytesPerGiB / 2) / k_BytesPerGiB; + if(hundredths > k_MaxOocSizeHundredths) + { + hundredths = k_MaxOocSizeHundredths; + } + if(bytes > 0 && hundredths == 0) + { + hundredths = 1; + } + return hundredths; + } + Preferences(); ~Preferences() noexcept; @@ -126,7 +191,11 @@ class SIMPLNX_EXPORT Preferences * @param name Preference key to update. * @param value New JSON value. * - * Updating the large-data threshold also recomputes memory defaults. + * Both out-of-core size thresholds accept unsigned JSON integers and + * nonnegative signed JSON integers. Changing Large Data Size recomputes + * memory defaults. + * Invalid thresholds throw + * std::invalid_argument before mutation. */ void setValue(const std::string& name, const nlohmann::json& value); @@ -175,15 +244,25 @@ class SIMPLNX_EXPORT Preferences * @return Error when the parent directory or output file cannot open. * * The method creates missing parent directories. Default values are not written. + * The file stores each out-of-core size threshold in GB under k_LargeDataSizeGB_Key or + * k_LargeDataStructureSizeGB_Key, as OocSizeBytesToHundredths(bytes) / 100. The byte keys are + * not written. All other explicit values are written unchanged. */ Result<> saveToFile(const std::filesystem::path& filepath) const; /** * @brief Reads explicit preferences from a JSON file. * @param filepath Source file path. - * @return Error when the file is absent, cannot open, or contains invalid JSON. + * @return Error when the file is absent, cannot open, contains invalid JSON, or has a non-object root. + * A valid result has one warning for each invalid out-of-core size threshold in the file. + * That threshold then uses its default. * - * The method migrates legacy cache and storage keys before updating memory defaults. + * For each out-of-core size threshold, a GB key takes precedence over the byte key that older + * versions write. The method does not validate that byte key. A GB value must be a finite + * JSON number from 0 to 17179869183.99. A legacy byte value must pass ValidateOocSizeValue(). + * The method stores a valid value in bytes on the 0.01-GB grid. A value above 0 that rounds + * to 0 becomes 0.01 GB. The method then migrates legacy cache and storage keys and updates + * memory defaults. The method does not write the file; the next saveToFile() call writes the GB keys. */ Result<> loadFromFile(const std::filesystem::path& filepath); @@ -219,7 +298,8 @@ class SIMPLNX_EXPORT Preferences * @brief Recomputes the default whole-data-structure size threshold. * * The threshold reserves two single-array limits for the operating system and - * application. Low-memory systems use half of physical RAM instead. + * application. Low-memory systems use half of physical RAM instead. The method rounds the + * threshold to the 0.01-GB grid, so a saved and reloaded default keeps its exact byte count. */ void updateMemoryDefaults(); diff --git a/src/simplnx/DataStructure/AbstractDataStore.hpp b/src/simplnx/DataStructure/AbstractDataStore.hpp index 6df7f06f77..15d22da8ff 100644 --- a/src/simplnx/DataStructure/AbstractDataStore.hpp +++ b/src/simplnx/DataStructure/AbstractDataStore.hpp @@ -6,6 +6,7 @@ #include "simplnx/Common/TypesUtility.hpp" #include "simplnx/DataStructure/IDataStore.hpp" +#include #include #include @@ -14,6 +15,8 @@ #include #include #include +#include +#include #include #include @@ -738,6 +741,32 @@ class AbstractDataStore : public IDataStore */ [[nodiscard]] virtual Result<> readExtentIntoBuffer(const Extent& extent, nonstd::span destination) const = 0; + /** + * @brief Optionally reads an extent with an explicit backend scratch allowance. + * + * The destination belongs to the caller and is outside the allowance. True + * means that every destination value is valid. A valid false result leaves + * the destination unchanged and performs no payload read. An invalid result + * can leave a prefix and requires the caller to discard the destination. + * The default implementation declines without invoking a payload reader. + * Mandatory empty Result controls are caller-funded ABI storage, outside the + * backend allowance. Iterator-debug builds can allocate fixed empty-vector + * bookkeeping. No payload, preparation or diagnostic text is allowed before + * admission. BoundedRead::EmptyResultControlBytes() describes reviewed controls. + * + * @param extent Supplies tuple-space coordinates in row-major order. + * @param destination Receives exactly the selected tuples and all components. + * @param scratchBudgetBytes Limits owned backend scratch, including diagnostics. + * @return Complete, optional unavailability, or a validation/read error. + */ + [[nodiscard]] virtual Result readExtentIntoBufferBounded(const Extent& extent, nonstd::span destination, uint64 scratchBudgetBytes) const + { + static_cast(extent); + static_cast(destination); + static_cast(scratchBudgetBytes); + return {false}; + } + /** * @brief Reads related N-dimensional extents into caller-owned storage. * @@ -1211,6 +1240,30 @@ class AbstractDataStore : public IDataStore { } + /** + * @brief Flushes storage through the legacy operation and reports failures. + * @return Success, error -272 for allocation failure, or -6070 for another exception. + * + * @note Diagnostic allocation can still throw std::bad_alloc. + */ + [[nodiscard]] virtual Result<> flushChecked() const + { + try + { + flush(); + return {}; + } catch(const std::bad_alloc& error) + { + return MakeErrorResult(-272, fmt::format("Cannot flush numeric storage: memory allocation failed: {}", error.what())); + } catch(const std::exception& error) + { + return MakeErrorResult(-6070, fmt::format("Cannot flush numeric storage: {}", error.what())); + } catch(...) + { + return MakeErrorResult(-6070, "Cannot flush numeric storage: unknown storage failure."); + } + } + /** * @brief Returns the approximate memory usage in bytes. * @return Approximate memory usage in bytes. @@ -1234,11 +1287,57 @@ class AbstractDataStore : public IDataStore */ virtual Result<> writeHdf5(HDF5::DatasetIO& dataset) const = 0; + /** + * @brief Returns the value that initializes elements when the store grows. + * @return Initialization value, or no value if growth uses the mudflap value. + */ + std::optional getInitValue() const + { + return m_InitValue; + } + + /** + * @brief Sets the policy for values added when the store grows. + * @param value Initialization value, or no value to use the mudflap value. + * @note Existing values do not change. Callers must exclude concurrent policy changes, resize and copying. + */ + void setInitValue(std::optional value) + { + m_InitValue = value; + } + + /** + * @brief Sets the value used when the store grows. + * @param value Value that initializes new elements. + * @note Existing values do not change. Callers must exclude concurrent policy changes, resize and copying. + */ + void setInitValue(T value) + { + m_InitValue = value; + } + protected: /** - * @brief Creates a data store without values. + * @brief Creates base state without allocating values. + * @param initValue Growth initializer; nullopt selects the diagnostic value. */ - AbstractDataStore() = default; + explicit AbstractDataStore(std::optional initValue = T{}) + : m_InitValue(initValue) + { + } + + /** + * @brief Resolves the value used for newly added elements. + * @return Explicit initializer, or GetMudflap() when the initializer is absent. + */ + T getGrowthInitValue() const + { + if(m_InitValue.has_value()) + { + return *m_InitValue; + } + return GetMudflap(); + } /** * @brief Copies base data-store state. @@ -1246,6 +1345,7 @@ class AbstractDataStore : public IDataStore */ AbstractDataStore(const AbstractDataStore& other) : IDataStore(other) + , m_InitValue(other.m_InitValue) { } @@ -1255,8 +1355,12 @@ class AbstractDataStore : public IDataStore */ AbstractDataStore(AbstractDataStore&& other) : IDataStore(std::move(other)) + , m_InitValue(std::move(other.m_InitValue)) { } + +private: + std::optional m_InitValue; }; using UInt8AbstractDataStore = AbstractDataStore; diff --git a/src/simplnx/DataStructure/BaseGroup.cpp b/src/simplnx/DataStructure/BaseGroup.cpp index a19058b4db..5b751a28a1 100644 --- a/src/simplnx/DataStructure/BaseGroup.cpp +++ b/src/simplnx/DataStructure/BaseGroup.cpp @@ -153,6 +153,11 @@ bool BaseGroup::insert(const std::weak_ptr& obj) return false; } +bool BaseGroup::acceptsChild(const DataObject* object) const +{ + return canInsert(object); +} + bool BaseGroup::remove(DataObject* obj) { if(obj == nullptr) diff --git a/src/simplnx/DataStructure/BaseGroup.hpp b/src/simplnx/DataStructure/BaseGroup.hpp index 12c0d582db..bb70b8c2a1 100644 --- a/src/simplnx/DataStructure/BaseGroup.hpp +++ b/src/simplnx/DataStructure/BaseGroup.hpp @@ -254,6 +254,15 @@ class SIMPLNX_EXPORT BaseGroup : public DataObject */ bool insert(const std::weak_ptr& obj); + /** + * @brief Checks the group's insertion rules without changing the group or object. + * @param object Supplies the candidate; null is rejected. + * @return True when the current virtual insertion rules accept the object. + * @pre The caller excludes concurrent changes to this group and the candidate. + * @note This query does not reserve a placement or prevent later insertion failure. + */ + [[nodiscard]] bool acceptsChild(const DataObject* object) const; + /** * @brief Attempts to remove the specified DataObject from the container. Returns * true if it succeeded. Returns false otherwise. diff --git a/src/simplnx/DataStructure/DataArray.hpp b/src/simplnx/DataStructure/DataArray.hpp index 24c76215db..cdb116273a 100644 --- a/src/simplnx/DataStructure/DataArray.hpp +++ b/src/simplnx/DataStructure/DataArray.hpp @@ -335,14 +335,6 @@ class DataArray : public IDataArray { return nullptr; } - // Preserve resident growth initialization when destination policy also selects memory. - const auto* sourceStorePtr = dynamic_cast*>(getDataStore()); - auto* destinationStorePtr = dynamic_cast*>(dataStore.get()); - if(sourceStorePtr != nullptr && destinationStorePtr != nullptr && sourceStorePtr->getStoreType() == IDataStore::StoreType::InMemory && - destinationStorePtr->getStoreType() == IDataStore::StoreType::InMemory) - { - destinationStorePtr->setInitValue(sourceStorePtr->getInitValue()); - } // Insertion assigns the destination identifier. std::shared_ptr> copy = std::shared_ptr>(new DataArray(dataStruct, copyPath.getTargetName(), dataStore)); if(dataStruct.insert(copy, copyPath.getParent())) @@ -827,6 +819,24 @@ class DataArray : public IDataArray m_DataStore->flush(); } + /** + * @brief Flushes the backing store and preserves its diagnostics. + * @return Store diagnostics, or error -6070 for missing storage or a failure without an error diagnostic. + */ + [[nodiscard]] Result<> flushChecked() const override + { + if(m_DataStore == nullptr) + { + return MakeErrorResult(-6070, fmt::format("Cannot flush numeric array '{}': its data store is null.", getName())); + } + auto result = m_DataStore->flushChecked(); + if(result.invalid() && result.errors().empty()) + { + result.errors().push_back({-6070, fmt::format("Cannot flush numeric array '{}': its data store failed without an error diagnostic.", getName())}); + } + return result; + } + /** * @brief Returns data-store memory usage in bytes. * @return Approximate memory usage in bytes. diff --git a/src/simplnx/DataStructure/DataObject.cpp b/src/simplnx/DataStructure/DataObject.cpp index d52f530536..5bc8788860 100644 --- a/src/simplnx/DataStructure/DataObject.cpp +++ b/src/simplnx/DataStructure/DataObject.cpp @@ -1,10 +1,15 @@ #include "DataObject.hpp" +#include "simplnx/Common/Result.hpp" #include "simplnx/DataStructure/BaseGroup.hpp" #include "simplnx/DataStructure/DataStructure.hpp" #include "simplnx/Utilities/StringUtilities.hpp" +#include + #include +#include +#include #include using namespace nx::core; @@ -341,6 +346,24 @@ void DataObject::flush() const { } +Result<> DataObject::flushChecked() const +{ + try + { + flush(); + return {}; + } catch(const std::bad_alloc& error) + { + return MakeErrorResult(-272, fmt::format("Cannot flush object '{}': memory allocation failed: {}", getName(), error.what())); + } catch(const std::exception& error) + { + return MakeErrorResult(-6070, fmt::format("Cannot flush object '{}': {}", getName(), error.what())); + } catch(...) + { + return MakeErrorResult(-6070, fmt::format("Cannot flush object '{}': unknown storage failure.", getName())); + } +} + uint64 DataObject::memoryUsage() const { return 0; diff --git a/src/simplnx/DataStructure/DataObject.hpp b/src/simplnx/DataStructure/DataObject.hpp index 6720a3edb4..8665e6d93a 100644 --- a/src/simplnx/DataStructure/DataObject.hpp +++ b/src/simplnx/DataStructure/DataObject.hpp @@ -1,5 +1,6 @@ #pragma once +#include "simplnx/Common/Result.hpp" #include "simplnx/Common/StringLiteral.hpp" #include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/Metadata.hpp" @@ -272,6 +273,14 @@ class SIMPLNX_EXPORT DataObject */ virtual void flush() const; + /** + * @brief Flushes the object through its legacy operation and reports failures. + * @return Success, error -272 for allocation failure, or -6070 for another exception. + * + * @note Diagnostic allocation can still throw std::bad_alloc. + */ + [[nodiscard]] virtual Result<> flushChecked() const; + /** * @brief Calculates and returns the memory usage of this DataObject in bytes. * @return Memory usage in bytes diff --git a/src/simplnx/DataStructure/DataStore.hpp b/src/simplnx/DataStructure/DataStore.hpp index 4ce60412f0..5cd6e45347 100644 --- a/src/simplnx/DataStructure/DataStore.hpp +++ b/src/simplnx/DataStructure/DataStore.hpp @@ -1,5 +1,7 @@ #pragma once +#include "simplnx/Common/BoundedRead.hpp" + #include "simplnx/Utilities/StoreCopyUtilities.hpp" #include "simplnx/Common/Bit.hpp" @@ -73,7 +75,7 @@ class DataStore : public AbstractDataStore /** * @brief Creates a one-component data store. * @param numTuples Number of tuples. - * @param initValue Optional value that initializes every element. + * @param initValue Initial value retained for future growth; nullopt leaves initial elements unwritten. */ DataStore(usize numTuples, std::optional initValue) : DataStore({numTuples}, {1}, initValue) @@ -84,24 +86,21 @@ class DataStore : public AbstractDataStore * @brief Creates a data store with a tuple and component shape. * @param tupleShape Tuple dimensions in slowest-to-fastest order. * @param componentShape Component dimensions in slowest-to-fastest order. - * @param initValue Optional value that initializes every element. + * @param initValue Initial value retained for future growth; nullopt leaves initial elements unwritten. */ DataStore(const ShapeType& tupleShape, const ShapeType& componentShape, std::optional initValue) - : parent_type() + : parent_type(initValue) , m_ComponentShape(componentShape) , m_TupleShape(tupleShape) , m_NumComponents(std::accumulate(m_ComponentShape.cbegin(), m_ComponentShape.cend(), static_cast(1), std::multiplies<>())) , m_NumTuples(std::accumulate(m_TupleShape.cbegin(), m_TupleShape.cend(), static_cast(1), std::multiplies<>())) - , m_InitValue(initValue) { - // new value_type[n] leaves the buffer uninitialized. std::make_unique would - // value-initialize it, adding a full pass over every byte of a store the caller is - // about to overwrite, and it would turn a read of a never-written element into a - // plausible zero instead of visible garbage. + // new value_type[n] leaves the buffer uninitialized. Value initialization would add a full pass before the caller overwrites these values. + // It would also hide reads of unwritten elements behind plausible zeros. m_Data = std::unique_ptr(new value_type[this->getSize()]); - if(m_InitValue.has_value()) + if(initValue.has_value()) { - std::fill_n(data(), this->getSize(), *m_InitValue); + std::fill_n(data(), this->getSize(), *initValue); } } @@ -115,15 +114,13 @@ class DataStore : public AbstractDataStore * @param componentShape Component dimensions in slowest-to-fastest order. */ DataStore(std::unique_ptr buffer, ShapeType tupleShape, ShapeType componentShape) - : parent_type() + : parent_type(GetMudflap()) , m_ComponentShape(std::move(componentShape)) , m_TupleShape(std::move(tupleShape)) , m_Data(std::move(buffer)) , m_NumComponents(std::accumulate(m_ComponentShape.cbegin(), m_ComponentShape.cend(), static_cast(1), std::multiplies<>())) , m_NumTuples(std::accumulate(m_TupleShape.cbegin(), m_TupleShape.cend(), static_cast(1), std::multiplies<>())) { - // Future growth needs a diagnostic value because the supplied buffer has no initialization value. - m_InitValue = GetMudflap(); } /** @@ -131,12 +128,11 @@ class DataStore : public AbstractDataStore * @param other Source data store. */ DataStore(const DataStore& other) - : parent_type() + : parent_type(other) , m_ComponentShape(other.m_ComponentShape) , m_TupleShape(other.m_TupleShape) , m_NumComponents(other.m_NumComponents) , m_NumTuples(other.m_NumTuples) - , m_InitValue(other.m_InitValue) { const usize count = other.getSize(); auto* data = new value_type[count]; @@ -149,13 +145,12 @@ class DataStore : public AbstractDataStore * @param other Source data store. */ DataStore(DataStore&& other) noexcept - : parent_type() + : parent_type(std::move(other)) , m_ComponentShape(std::move(other.m_ComponentShape)) , m_TupleShape(std::move(other.m_TupleShape)) , m_Data(std::move(other.m_Data)) , m_NumComponents(std::move(other.m_NumComponents)) , m_NumTuples(std::move(other.m_NumTuples)) - , m_InitValue(other.m_InitValue) { } @@ -173,7 +168,7 @@ class DataStore : public AbstractDataStore m_Data = std::move(rhs.m_Data); m_NumComponents = rhs.m_NumComponents; m_NumTuples = rhs.m_NumTuples; - m_InitValue = std::move(rhs.m_InitValue); + this->setInitValue(rhs.getInitValue()); return *this; } @@ -234,33 +229,6 @@ class DataStore : public AbstractDataStore return {}; } - /** - * @brief Returns the value that initializes elements when the store grows. - * @return Initialization value, or no value if growth uses the mudflap value. - */ - std::optional getInitValue() const - { - return m_InitValue; - } - - /** - * @brief Sets the policy for values added when the store grows. - * @param value Initialization value, or no value to use the mudflap value. - */ - void setInitValue(std::optional value) - { - m_InitValue = value; - } - - /** - * @brief Sets the value used when the store grows. - * @param value Value that initializes new elements. - */ - void setInitValue(T value) - { - m_InitValue = value; - } - /** * @brief Changes the tuple shape. * @@ -309,9 +277,8 @@ class DataStore : public AbstractDataStore return {}; } - // new value_type[n] leaves the buffer uninitialized. Every element is then either - // copied from the previous buffer or set to the initialization or mudflap value, so - // value-initializing here would only repeat that work. + // new value_type[n] leaves the buffer uninitialized. The following loops copy the retained prefix and initialize the new tail. + // Value initialization would repeat that work. std::unique_ptr data(new value_type[newSize]); if(m_Data != nullptr) { @@ -320,7 +287,7 @@ class DataStore : public AbstractDataStore data[valueIndex] = m_Data[valueIndex]; } - const T initValue = m_InitValue.has_value() ? *m_InitValue : GetMudflap(); + const T initValue = this->getGrowthInitValue(); for(usize valueIndex = oldSize; valueIndex < newSize; ++valueIndex) { data[valueIndex] = initValue; @@ -495,6 +462,68 @@ class DataStore : public AbstractDataStore } } + /** + * @brief Reads a validated extent directly into caller storage within a fixed allowance. + * @param extent Supplies tuple-space coordinates. + * @param destination Receives every component of each selected tuple. + * @param scratchBudgetBytes Funds the fixed diagnostic carrier (4096 bytes). + * @return True on completion, false below the minimum or for bool, or an invalid result. + */ + [[nodiscard]] Result readExtentIntoBufferBounded(const Extent& extent, nonstd::span destination, uint64 scratchBudgetBytes) const override + { + if(scratchBudgetBytes < BoundedRead::k_DiagnosticBytes || !BoundedRead::DiagnosticRepresentationSupported()) + { + return {false}; + } + BoundedRead::Diagnostic diagnostic; + if constexpr(std::is_same_v) + { + return diagnostic.finish(diagnostic.set(BoundedRead::Unsupported, "Bounded numeric reads do not support bool stores.")); + } + uint64 components = 1; + for(const auto size : getComponentShape()) + { + if(size == 0 || !BoundedRead::Multiply(components, static_cast(size), components)) + { + return diagnostic.finish(diagnostic.set(BoundedRead::InvalidExtent, "Bounded read component shape is empty or overflows uint64.")); + } + } + uint64 selectedTuples = 0; + const auto status = BoundedRead::ValidateExtent(extent, getTupleShape(), components, destination.size(), selectedTuples, diagnostic, sizeof(T)); + if(status != BoundedRead::Status::Complete) + { + return diagnostic.finish(status); + } + if(m_Data == nullptr) + { + return diagnostic.finish(diagnostic.set(BoundedRead::ReadFailure, "Bounded resident read has no allocated source buffer.")); + } + // All products are checked above. Map each selected tuple without coordinate allocation. + const auto& shape = getTupleShape(); + for(uint64 outputTuple = 0; outputTuple < selectedTuples; ++outputTuple) + { + uint64 remaining = outputTuple; + uint64 sourceTuple = 0; + uint64 sourceStride = 1; + for(usize reverse = shape.size(); reverse != 0; --reverse) + { + const usize dimension = reverse - 1; + const uint64 count = (extent.max[dimension] - extent.min[dimension]) / extent.stride[dimension] + 1; + const uint64 coordinate = extent.min[dimension] + (remaining % count) * extent.stride[dimension]; + remaining /= count; + sourceTuple += coordinate * sourceStride; + sourceStride *= static_cast(shape[dimension]); + } + const usize sourceOffset = static_cast(sourceTuple * components); + const usize outputOffset = static_cast(outputTuple * components); + for(usize component = 0; component < static_cast(components); ++component) + { + destination[outputOffset + component] = m_Data[sourceOffset + component]; + } + } + return {true}; + } + /** * @brief Reads a tuple-space extent into caller-owned storage. * @@ -1076,7 +1105,6 @@ class DataStore : public AbstractDataStore std::unique_ptr m_Data = nullptr; usize m_NumComponents = {0}; usize m_NumTuples = {0}; - std::optional m_InitValue; }; /** diff --git a/src/simplnx/DataStructure/DataStructure.cpp b/src/simplnx/DataStructure/DataStructure.cpp index 628afcef51..06d22f81cd 100644 --- a/src/simplnx/DataStructure/DataStructure.cpp +++ b/src/simplnx/DataStructure/DataStructure.cpp @@ -1,8 +1,11 @@ #include "DataStructure.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/ScopeGuard.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/BaseGroup.hpp" #include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/DataStructure/DataObject.hpp" #include "simplnx/DataStructure/Geometry/IGeometry.hpp" #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/DataStructure/INeighborList.hpp" @@ -18,11 +21,28 @@ #include "simplnx/Utilities/DataArrayUtilities.hpp" #include "simplnx/Utilities/DataGroupUtilities.hpp" -#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include #include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; namespace { @@ -44,6 +64,32 @@ std::shared_ptr& DefaultFormatResolver namespace nx::core { +/** + * @struct ImportPublicationTestState + * @brief Keeps action-local faults and failed-transaction owners in DataStructure. + */ +struct DataStructure::ImportPublicationTestState +{ + ImportPublicationTestState() = default; + ~ImportPublicationTestState() noexcept = default; + ImportPublicationTestState(const ImportPublicationTestState&) = delete; + ImportPublicationTestState(ImportPublicationTestState&&) = delete; + ImportPublicationTestState& operator=(const ImportPublicationTestState&) = delete; + ImportPublicationTestState& operator=(ImportPublicationTestState&&) = delete; + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + ImportPublicationTestControl::Fault Fault = ImportPublicationTestControl::Fault::None; + bool Consumed = false; + bool RefuseCleanup = false; + bool CleanupConsumed = false; + std::shared_ptr RetainedOwner; + std::vector RetainedRecords; + DataStructure* Destination = nullptr; + DataObject::IdType SavedNextId = 0; + DataObject::IdType ReservedNextId = 0; +#endif +}; + DataStructure::DataStructure() : m_IsValid(true) { @@ -110,6 +156,11 @@ const IDataStoreFormatResolver& DataStructure::formatResolver() const return *DefaultFormatResolverRef(); } +std::shared_ptr DataStructure::getDefaultFormatResolver() +{ + return DefaultFormatResolverRef(); +} + void DataStructure::setDefaultFormatResolver(std::shared_ptr resolver) { if(resolver != nullptr) @@ -669,6 +720,499 @@ bool DataStructure::insert(const std::shared_ptr& dataObject, const return insertIntoParent(dataObject, parentGroup); } +DataStructure::ImportPublicationCheckpoint::ImportPublicationCheckpoint(const DataStructure& destination) noexcept +: m_Destination(&destination) +, m_NextId(destination.m_NextId) +{ +} + +bool DataStructure::ImportPublicationCheckpoint::belongsTo(const DataStructure& destination) const noexcept +{ + return m_Destination == &destination; +} + +DataObject::IdType DataStructure::ImportPublicationCheckpoint::nextId() const noexcept +{ + return m_NextId; +} + +DataObject::IdType DataStructure::getLiveObjectCount() const +{ + DataObject::IdType count = 0; + for(const auto& entry : m_DataObjects) + { + if(!entry.second.expired()) + { + ++count; + } + } + return count; +} + +std::set DataStructure::getOccupiedObjectIds() const +{ + std::set occupiedIds; + for(const auto& entry : m_DataObjects) + { + occupiedIds.insert(entry.first); + } + // Registry enumeration omits hierarchy-only placements. Inspect actual maps before copying the destination. + std::vector pendingMaps{&getDataMap()}; + std::unordered_set visitedObjects; + while(!pendingMaps.empty()) + { + const auto* mapPtr = pendingMaps.back(); + pendingMaps.pop_back(); + for(const auto& [id, owner] : *mapPtr) + { + occupiedIds.insert(id); + if(owner == nullptr) + { + continue; + } + occupiedIds.insert(owner->getId()); + if(visitedObjects.insert(owner.get()).second) + { + if(const auto* groupPtr = dynamic_cast(owner.get()); groupPtr != nullptr) + { + pendingMaps.push_back(&groupPtr->getDataMap()); + } + } + } + } + + return occupiedIds; +} + +Result<> DataStructure::prepareImportPublicationRecords(DataStructure& prepared, const std::vector& paths, const fs::path& sourceFile, std::vector& records, + DataObject::IdType& reservedNextId) +{ + constexpr auto maximumId = std::numeric_limits::max(); + records.reserve(paths.size()); + std::map> publicationGroups; + std::set selectedPaths; + std::unordered_map> selectedPlacements; + std::optional> occupiedIds; + for(const auto& path : paths) + { + const auto object = prepared.getSharedData(path); + if(object == nullptr) + { + // Recovery may remove an invalid leaf with a warning. Preserve that partial-recovery contract. + continue; + } + if(containsData(path)) + { + return MakeErrorResult(-6203, fmt::format("Cannot publish '{}' from '{}': the destination path already exists. Rename it or exclude this path.", path.toString(), sourceFile.string())); + } + if(!selectedPaths.insert(path).second) + { + return MakeErrorResult(-6203, fmt::format("Cannot publish '{}' from '{}': the same path is selected more than once.", path.toString(), sourceFile.string())); + } + if(!occupiedIds.has_value()) + { + occupiedIds = getOccupiedObjectIds(); + } + ImportPublicationRecord record; + record.Owner = makeImportPublicationCopy(*object); + record.Id = object->getId(); + record.Path = path; + if(record.Owner == nullptr || record.Id == 0 || record.Id == maximumId || !occupiedIds->insert(record.Id).second) + { + return MakeErrorResult(-6213, fmt::format("Cannot publish '{}' from '{}': identifier {} or its prepared owner is invalid.", path.toString(), sourceFile.string(), record.Id)); + } + const auto parentPath = path.getParent(); + if(!parentPath.empty()) + { + auto parent = publicationGroups.find(parentPath); + record.Parent = parent == publicationGroups.end() ? std::dynamic_pointer_cast(getSharedData(parentPath)) : parent->second; + if(record.Parent == nullptr) + { + return MakeErrorResult(-6202, fmt::format("Cannot publish '{}': destination parent '{}' is unavailable.", path.toString(), parentPath.toString())); + } + record.ParentId = record.Parent->getId(); + } + // Distinct paths through parent aliases can refer to the same destination child name. + if(!selectedPlacements[record.Parent.get()].insert(record.Owner->getName()).second) + { + return MakeErrorResult(-6203, fmt::format("Cannot publish '{}' from '{}': its destination placement is selected more than once. Select only one object named '{}' in that parent.", + path.toString(), sourceFile.string(), record.Owner->getName())); + } + record.RemovalMessage = std::make_shared(this, record.Id, record.Owner->getName()); + reservedNextId = std::max(reservedNextId, record.Id + 1); + if(auto group = std::dynamic_pointer_cast(record.Owner); group != nullptr) + { + publicationGroups.emplace(path, std::move(group)); + } + records.push_back(std::move(record)); + } + return {}; +} + +bool DataStructure::rollbackImportPublication(std::vector& records, DataObject::IdType savedNextId, DataObject::IdType reservedNextId, + [[maybe_unused]] ImportPublicationTestState* testState, [[maybe_unused]] bool allowRefusal) +{ + bool complete = true; + // Finish all independent structural cleanup before any observer can change the destination. + for(usize index = records.size(); index > 0; --index) + { + auto& record = records[index - 1]; + if(record.Owner == nullptr) + { + continue; + } +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + if(allowRefusal && testState != nullptr && testState->RefuseCleanup && index == 2 && record.Stage != ImportInsertionStage::None) + { + testState->CleanupConsumed = true; + record.Cleanup = ImportCleanupStatus::Refused; + complete = false; + continue; + } +#endif + record.Cleanup = rollbackImportedObject(record); + complete = complete && record.Cleanup == ImportCleanupStatus::Complete; + } + for(auto& record : std::views::reverse(records)) + { + if(record.Cleanup == ImportCleanupStatus::Complete && record.NotifyRemoval && !record.NotificationAttempted) + { + record.NotificationAttempted = true; + try + { + notify(record.RemovalMessage); + } catch(...) + { + record.NotificationFailed = true; + } + } + } + for(auto& record : records) + { + if(record.Cleanup == ImportCleanupStatus::Complete) + { + record.Owner.reset(); + record.Parent.reset(); + } + } + // A removal observer can create unrelated objects. Preserve its advanced counter. + if(complete && m_NextId == reservedNextId) + { + m_NextId = savedNextId; + } + return complete; +} + +void DataStructure::appendImportCleanupDiagnostics(Result<>& result, const std::vector& records, const fs::path& sourceFile) +{ + for(const auto& record : records) + { + if(record.Cleanup != ImportCleanupStatus::Complete) + { + result.errors().push_back( + {.code = -6215, + .message = fmt::format("Import rollback retained '{}' (identifier {}) from '{}': {}. The destination ID counter remains advanced.", record.Path.toString(), record.Id, sourceFile.string(), + record.Cleanup == ImportCleanupStatus::Children ? "selected children remain attached" : "exact placement cleanup did not complete")}); + } + if(record.NotificationFailed) + { + result.errors().push_back({.code = -6216, .message = fmt::format("Import rollback removed '{}' (identifier {}), but a removal notification failed.", record.Path.toString(), record.Id)}); + } + } +} + +void DataStructure::retainImportFailure([[maybe_unused]] ImportPublicationTestState* testState, [[maybe_unused]] std::vector& records, + [[maybe_unused]] DataObject::IdType savedNextId, [[maybe_unused]] DataObject::IdType reservedNextId) +{ +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + if(testState != nullptr) + { + testState->Destination = this; + testState->SavedNextId = savedNextId; + testState->ReservedNextId = reservedNextId; + testState->RetainedRecords = std::move(records); + } +#endif +} + +Result<> DataStructure::publishImportedObjects(DataStructure& prepared, const std::vector& paths, const fs::path& sourceFile, const ImportPublicationCheckpoint& checkpoint) +{ + return publishImportedObjectsImpl(prepared, paths, sourceFile, checkpoint, nullptr); +} + +Result<> DataStructure::publishImportedObjectsImpl(DataStructure& prepared, const std::vector& paths, const fs::path& sourceFile, const ImportPublicationCheckpoint& checkpoint, + ImportPublicationTestState* testState) +{ + if(!checkpoint.belongsTo(*this)) + { + return MakeErrorResult( + -6217, + fmt::format("Cannot publish selected objects from '{}': the publication checkpoint belongs to another destination. Capture a checkpoint from this destination before preparing the import.", + sourceFile.string())); + } + if(m_NextId != checkpoint.nextId()) + { + return MakeErrorResult( + -6218, + fmt::format( + "Cannot publish selected objects from '{}': the destination next identifier is {}, but the publication checkpoint recorded {}. Capture a new checkpoint and prepare the import again.", + sourceFile.string(), m_NextId, checkpoint.nextId())); + } +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + using PublicationFaultType = ImportPublicationTestControl::Fault; +#endif + const auto savedNextId = checkpoint.nextId(); + auto reservedNextId = savedNextId; + std::vector records; + auto importResult = prepareImportPublicationRecords(prepared, paths, sourceFile, records, reservedNextId); + if(importResult.invalid() || records.empty()) + { + return importResult; + } + + // Publication owners, records, and removal messages are ready before the first destination mutation. + m_NextId = reservedNextId; + usize insertionIndex = 0; + try + { +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + if(testState != nullptr && testState->Fault == PublicationFaultType::BeforePublication) + { + testState->Consumed = true; + testState->Fault = PublicationFaultType::None; + throw std::runtime_error("injected import failure before publication"); + } +#endif + for(; insertionIndex < records.size(); ++insertionIndex) + { + auto failAfter = ImportInsertionStage::None; + bool rejectBeforeInsert = false; +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + if(testState != nullptr && insertionIndex == 1) + { + testState->RetainedOwner = records[insertionIndex].Owner; + if(testState->Fault == PublicationFaultType::AfterHierarchy) + { + failAfter = ImportInsertionStage::Hierarchy; + } + else if(testState->Fault == PublicationFaultType::BeforeTracking) + { + failAfter = ImportInsertionStage::Parent; + } + else if(testState->Fault == PublicationFaultType::ReturnFalse) + { + testState->Consumed = true; + testState->Fault = PublicationFaultType::None; + rejectBeforeInsert = true; + } + } +#endif + if(!insertImportedObject(records[insertionIndex], failAfter, rejectBeforeInsert)) + { + throw std::runtime_error("destination insertion rejected the prepared object"); + } +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + if(testState != nullptr && insertionIndex == 1 && (testState->Fault == PublicationFaultType::AfterSecondInsert || testState->Fault == PublicationFaultType::AllocationAfterSecond)) + { + const bool allocation = testState->Fault == PublicationFaultType::AllocationAfterSecond; + testState->Fault = PublicationFaultType::None; + testState->Consumed = true; + if(allocation) + { + throw std::bad_alloc(); + } + throw std::runtime_error("injected import failure after second insertion"); + } +#endif + } + } catch(const std::bad_alloc&) + { + rollbackImportPublication(records, savedNextId, reservedNextId, testState, true); + retainImportFailure(testState, records, savedNextId, reservedNextId); + throw; + } catch(const std::exception& error) + { +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + if(testState != nullptr && insertionIndex < records.size() && + ((testState->Fault == PublicationFaultType::AfterHierarchy && records[insertionIndex].Stage == ImportInsertionStage::Hierarchy) || + (testState->Fault == PublicationFaultType::BeforeTracking && records[insertionIndex].Stage == ImportInsertionStage::Parent))) + { + testState->Consumed = true; + testState->Fault = PublicationFaultType::None; + } +#endif + rollbackImportPublication(records, savedNextId, reservedNextId, testState, true); + const auto& failed = records[std::min(insertionIndex, records.size() - 1)]; + importResult = MergeResults(std::move(importResult), + MakeErrorResult(-6214, fmt::format("Cannot publish '{}' (identifier {}) from '{}': {}", failed.Path.toString(), failed.Id, sourceFile.string(), error.what()))); + appendImportCleanupDiagnostics(importResult, records, sourceFile); + retainImportFailure(testState, records, savedNextId, reservedNextId); + return importResult; + } catch(...) + { + rollbackImportPublication(records, savedNextId, reservedNextId, testState, true); + importResult = MergeResults(std::move(importResult), MakeErrorResult(-6214, fmt::format("Cannot publish selected objects from '{}': an unknown insertion failure occurred.", sourceFile.string()))); + appendImportCleanupDiagnostics(importResult, records, sourceFile); + retainImportFailure(testState, records, savedNextId, reservedNextId); + return importResult; + } + return importResult; +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +DataStructure::ImportPublicationTestControl::ImportPublicationTestControl() = default; +DataStructure::ImportPublicationTestControl::~ImportPublicationTestControl() noexcept = default; + +void DataStructure::ImportPublicationTestControl::setFault(Fault fault, bool refuseCleanup) +{ + m_State = std::make_unique(); + m_State->Fault = fault; + m_State->RefuseCleanup = refuseCleanup; +} + +bool DataStructure::ImportPublicationTestControl::publicationFaultConsumed() const +{ + return m_State != nullptr && m_State->Consumed; +} + +bool DataStructure::ImportPublicationTestControl::cleanupFaultConsumed() const +{ + return m_State != nullptr && m_State->CleanupConsumed; +} + +std::shared_ptr DataStructure::ImportPublicationTestControl::retainedOwner() const +{ + return m_State == nullptr ? nullptr : m_State->RetainedOwner; +} + +Result<> DataStructure::ImportPublicationTestControl::publish(DataStructure& destination, DataStructure& prepared, const std::vector& paths, const fs::path& sourceFile, + const ImportPublicationCheckpoint& checkpoint) +{ + return destination.publishImportedObjectsImpl(prepared, paths, sourceFile, checkpoint, m_State.get()); +} + +Result<> DataStructure::ImportPublicationTestControl::cleanup(DataStructure& destination) +{ + if(m_State == nullptr || m_State->Destination != &destination) + { + return MakeErrorResult(-6215, "The selected import has no retained rollback records for this destination."); + } + if(!destination.rollbackImportPublication(m_State->RetainedRecords, m_State->SavedNextId, m_State->ReservedNextId, m_State.get(), false)) + { + return MakeErrorResult(-6215, "The selected import still has an exact-object cleanup conflict."); + } + return {}; +} +#endif + +std::shared_ptr DataStructure::makeImportPublicationCopy(DataObject& source) +{ + auto* associationPtr = source.m_DataStructure; + source.DataObject::setDataStructure(nullptr); + const auto restore = MakeScopeGuard([&source, associationPtr]() noexcept { source.DataObject::setDataStructure(associationPtr); }); + std::unique_ptr copy(source.shallowCopy()); + if(copy == nullptr) + { + return {}; + } + if(auto* groupPtr = dynamic_cast(copy.get()); groupPtr != nullptr) + { + groupPtr->clear(); + } + copy->DataObject::setDataStructure(nullptr); + copy->m_ParentList.clear(); + return {std::move(copy)}; +} + +bool DataStructure::insertImportedObject(ImportPublicationRecord& record, ImportInsertionStage failAfter, bool rejectBeforeInsert) +{ + if(rejectBeforeInsert || record.Owner == nullptr || record.Owner->getId() != record.Id || m_DataObjects.contains(record.Id)) + { + return false; + } + if(record.Parent != nullptr && (record.Parent->getId() != record.ParentId || !record.Parent->acceptsChild(record.Owner.get()))) + { + return false; + } + auto& map = record.Parent == nullptr ? m_RootGroup : record.Parent->getDataMap(); + if(map.find(record.Id) != map.end() || !map.insert(record.Owner)) + { + return false; + } + record.Stage = ImportInsertionStage::Hierarchy; + if(failAfter == record.Stage) + { + throw std::runtime_error("injected import failure after hierarchy placement"); + } + if(record.Parent != nullptr) + { + record.Owner->addParent(record.Parent.get()); + } + record.Stage = ImportInsertionStage::Parent; + if(failAfter == record.Stage) + { + throw std::runtime_error("injected import failure before weak-index registration"); + } + trackDataObject(record.Owner); + record.Stage = ImportInsertionStage::Registered; + return true; +} + +DataStructure::ImportCleanupStatus DataStructure::rollbackImportedObject(ImportPublicationRecord& record) noexcept +{ + auto& object = *record.Owner; + if(object.getId() != record.Id || (record.Parent != nullptr && record.Parent->getId() != record.ParentId)) + { + return ImportCleanupStatus::Conflict; + } + auto& map = record.Parent == nullptr ? m_RootGroup : record.Parent->getDataMap(); + const auto placement = map.find(record.Id); + if(placement != map.end() && placement->second.get() != &object) + { + return ImportCleanupStatus::Conflict; + } + for(const auto parentId : object.m_ParentList) + { + if(record.Parent == nullptr || parentId != record.ParentId) + { + return ImportCleanupStatus::Conflict; + } + } + if(record.Parent != nullptr && m_RootGroup.find(record.Id) != m_RootGroup.end()) + { + return ImportCleanupStatus::Conflict; + } + if(const auto* groupPtr = dynamic_cast(&object); groupPtr != nullptr && !groupPtr->empty()) + { + return ImportCleanupStatus::Children; + } + const auto indexed = m_DataObjects.find(record.Id); + if(indexed != m_DataObjects.end() && indexed->second.lock().get() != &object) + { + // A foreign weak entry must survive. A locally unowned shell must still leave scope without a stale callback. + if(placement == map.end() && object.m_ParentList.empty()) + { + object.DataObject::setDataStructure(nullptr); + } + return ImportCleanupStatus::Conflict; + } + if(placement != map.end()) + { + map.erase(placement); + } + if(record.Parent != nullptr) + { + object.m_ParentList.remove(record.ParentId); + } + if(indexed != m_DataObjects.end()) + { + m_DataObjects.erase(indexed); + } + record.NotifyRemoval = object.m_DataStructure == this; + object.DataObject::setDataStructure(nullptr); + return ImportCleanupStatus::Complete; +} + DataObject::IdType DataStructure::getNextId() const { return m_NextId; @@ -988,6 +1532,88 @@ void DataStructure::flush() const } } +Result<> DataStructure::flushChecked() const +{ + ErrorCollection errors; + WarningCollection warnings; + for(const auto& [id, weakObject] : m_DataObjects) + { + const auto object = weakObject.lock(); + if(object == nullptr) + { + continue; + } + + Result<> objectResult; + try + { + objectResult = object->flushChecked(); + } catch(const std::bad_alloc& error) + { + objectResult = MakeErrorResult(-272, fmt::format("Memory allocation failed during checked flush: {}", error.what())); + } catch(const std::exception& error) + { + objectResult = MakeErrorResult(-6070, fmt::format("Checked flush failed: {}", error.what())); + } catch(...) + { + objectResult = MakeErrorResult(-6070, "Checked flush failed: unknown storage failure."); + } + if(objectResult.invalid() && objectResult.errors().empty()) + { + objectResult.errors().push_back({.code = -6070, .message = "Checked flush failed without an error diagnostic."}); + } + if(objectResult.valid() && objectResult.warnings().empty()) + { + continue; + } + + // Clean objects avoid path construction. The ID index visits linked objects only once. + std::string context = fmt::format("Checked flush for object '{}' (ID {})", object->getName(), id); + const auto paths = object->getDataPaths(); + if(!paths.empty()) + { + context += " at paths ["; + for(usize pathIdx = 0; pathIdx < paths.size(); ++pathIdx) + { + if(pathIdx != 0) + { + context += ", "; + } + context += fmt::format("'{}'", paths[pathIdx].toString()); + } + context += "]"; + } + + // Normal vector growth avoids repeatedly reallocating all earlier diagnostics. + if(objectResult.invalid()) + { + for(auto& error : objectResult.errors()) + { + error.message = fmt::format("{}: {}", context, error.message); + errors.push_back(std::move(error)); + } + } + for(auto& warning : objectResult.warnings()) + { + warning.message = fmt::format("{}: {}", context, warning.message); + warnings.push_back(std::move(warning)); + } + if(objectResult.invalid()) + { + // Stop at the first failed object. Later objects keep their unflushed state for a later flush. + break; + } + } + + Result<> result; + if(!errors.empty()) + { + result.m_Expected = nonstd::make_unexpected(std::move(errors)); + } + result.warnings() = std::move(warnings); + return result; +} + uint64 DataStructure::memoryUsage() const { uint64 memory = 0; diff --git a/src/simplnx/DataStructure/DataStructure.hpp b/src/simplnx/DataStructure/DataStructure.hpp index bdcdcacef1..62459224f2 100644 --- a/src/simplnx/DataStructure/DataStructure.hpp +++ b/src/simplnx/DataStructure/DataStructure.hpp @@ -107,10 +107,17 @@ class SIMPLNX_EXPORT DataStructure */ const IDataStoreFormatResolver& formatResolver() const; + /** + * @brief Retains the current process-wide storage-format resolver. + * @return Shared owner that keeps the resolver alive after the default changes. + * @pre Do not call while another thread changes the default resolver. + */ + static std::shared_ptr getDefaultFormatResolver(); + /** * @brief Sets the process-wide storage-format resolver. * @param resolver Has no effect when null. - * @pre Install the resolver before concurrent DataStructure use. + * @pre Do not replace the resolver during concurrent resolver access. */ static void setDefaultFormatResolver(std::shared_ptr resolver); @@ -639,6 +646,17 @@ class SIMPLNX_EXPORT DataStructure */ void flush() const; + /** + * @brief Flushes live objects once each in identifier order and stops at the first failed object. + * @return Warnings from the objects flushed so far, plus the failed object's errors, each with its name, identifier, and paths. + * + * @pre The caller excludes concurrent mutations. + * + * @note Objects after the first failure are not flushed by this call. Their unwritten data stays in memory + * for a later flush; nothing is discarded. A failure without error diagnostics receives error -6070. + */ + [[nodiscard]] Result<> flushChecked() const; + uint64 memoryUsage() const; /** @@ -654,6 +672,86 @@ class SIMPLNX_EXPORT DataStructure Result<> validateAttributeMatrices() const; + /** + * @class ImportPublicationCheckpoint + * @brief Captures the destination identity and next ID before import preparation. + * + * The checkpoint owns no destination and reserves no IDs. Publication rejects + * a foreign destination or a changed counter. A complete rollback permits reuse. + * The destination must remain alive at the same address. Explicit counter + * rewrites or destination replacement end the checkpoint lifetime. + */ + class SIMPLNX_EXPORT ImportPublicationCheckpoint + { + public: + /** + * @brief Captures the current destination counter without changing it. + * @param destination Remains alive through publication and any explicit cleanup. + * @pre The caller excludes concurrent destination mutation. + */ + explicit ImportPublicationCheckpoint(const DataStructure& destination) noexcept; + ~ImportPublicationCheckpoint() noexcept = default; + ImportPublicationCheckpoint(const ImportPublicationCheckpoint&) = default; + ImportPublicationCheckpoint(ImportPublicationCheckpoint&&) noexcept = default; + ImportPublicationCheckpoint& operator=(const ImportPublicationCheckpoint&) = default; + ImportPublicationCheckpoint& operator=(ImportPublicationCheckpoint&&) noexcept = default; + + /** + * @brief Checks the captured destination address without dereferencing it. + * @param destination Supplies the live publication target. + * @return True when the target has the captured address. + */ + [[nodiscard]] bool belongsTo(const DataStructure& destination) const noexcept; + [[nodiscard]] DataObject::IdType nextId() const noexcept; + + private: + const DataStructure* m_Destination = nullptr; + DataObject::IdType m_NextId = 0; + }; + + class ImportPublicationTestControl; + + /** + * @brief Counts live weak-index entries without counting hierarchy aliases twice. + * @return Number of nonexpired entries that resetIds renumbers. + * @pre The caller excludes concurrent mutations and object destruction. + */ + [[nodiscard]] DataObject::IdType getLiveObjectCount() const; + + /** + * @brief Collects IDs occupied by the weak index or actual hierarchy placements. + * @return Independent set containing expired index keys, hierarchy keys, and owner IDs. + * @pre The caller excludes concurrent mutations and object destruction. + */ + [[nodiscard]] std::set getOccupiedObjectIds() const; + + /** + * @brief Creates a detached shell whose failed construction cannot notify through the source association. + * @param source Supplies an object whose children remain strongly owned. + * + * @return Independent object shell sharing only its value store. + * @pre The caller excludes concurrent access to the source structure. + */ + [[nodiscard]] static std::shared_ptr makeImportPublicationCopy(DataObject& source); + + /** + * @brief Publishes selected prepared shells and rolls back exact placements on failure. + * @param prepared Owns completed import values; publication shares their stores. + * @param paths Lists selected paths in parent-before-child order. + * @param sourceFile Identifies the import in diagnostics; this method does not read it. + * @param checkpoint Captures this destination before preparation starts. + * @return Import diagnostics; -6217 rejects a foreign checkpoint and -6218 rejects a stale counter. + * @throws std::bad_alloc After rollback when publication allocation fails. + * @pre The caller serializes access to both structures and preserves the checkpoint lifetime. + * @post Rejected checkpoints leave both structures and test fault state unchanged. + * @post The returned Result contains no warnings; the caller retains warnings from import preparation. + * + * All publication owners and removal messages exist before destination mutation. + * Structural cleanup precedes removal notifications. Callback-created objects + * retain their advanced counter. Incomplete cleanup keeps reserved IDs and reports -6215. + */ + [[nodiscard]] Result<> publishImportedObjects(DataStructure& prepared, const std::vector& paths, const std::filesystem::path& sourceFile, const ImportPublicationCheckpoint& checkpoint); + protected: /** * @brief Returns and advances the next object identifier. @@ -672,6 +770,123 @@ class SIMPLNX_EXPORT DataStructure void setData(DataObject::IdType identifier, std::shared_ptr dataObject); private: + /** + * @enum ImportInsertionStage + * @brief Identifies the last completed mutation in one import insertion. + */ + enum class ImportInsertionStage : uint8 + { + None, ///< No hierarchy placement exists. + Hierarchy, ///< The hierarchy owns the object; its parent list can still be empty. + Parent, ///< The expected parent list is complete. + Registered ///< The weak index and destination association are complete. + }; + + /** + * @enum ImportCleanupStatus + * @brief Reports structural cleanup without allocating diagnostics. + */ + enum class ImportCleanupStatus : uint8 + { + Complete, ///< The exact object is detached and disarmed. + Conflict, ///< Identity or placement differs from the publication record. + Children, ///< A group still owns objects that cleanup must preserve. + Refused ///< A test fault retains the exact recorded placement. + }; + + /** + * @struct ImportPublicationRecord + * @brief Keeps exact owners and preallocated diagnostics alive through import rollback. + */ + struct ImportPublicationRecord + { + std::shared_ptr Owner; + std::shared_ptr Parent; + DataObject::IdType Id = 0; + DataObject::IdType ParentId = 0; + DataPath Path; + std::shared_ptr RemovalMessage; + ImportInsertionStage Stage = ImportInsertionStage::None; + ImportCleanupStatus Cleanup = ImportCleanupStatus::Complete; + bool NotifyRemoval = false; + bool NotificationAttempted = false; + bool NotificationFailed = false; + }; + + /** + * @brief Inserts one prechecked import object without ordinary add notifications. + * @param record Supplies exact owners and receives the completed mutation stage. + * @param failAfter + * Selects a private transaction fault, or None for normal insertion. + * @param rejectBeforeInsert Selects a returned-failure test boundary before mutation. + * @return True when hierarchy placement and registration complete. + */ + bool insertImportedObject(ImportPublicationRecord& record, ImportInsertionStage failAfter, bool rejectBeforeInsert); + + /** + * @brief Detaches only the recorded import placement and matching weak entry. + * @param record Keeps all owners alive and receives notification eligibility. + * @return Structural outcome; + * no notification or diagnostic allocation occurs here. + */ + ImportCleanupStatus rollbackImportedObject(ImportPublicationRecord& record) noexcept; + + struct ImportPublicationTestState; + + /** + * @brief Builds exact publication records without changing the destination. + * @param prepared Owns completed import shells. + * @param paths Lists selected paths in parent-before-child order. + * @param sourceFile Supplies diagnostic context. + * @param records Receives exact owners and preallocated removal messages. + * @param reservedNextId Receives the checked high-water counter. + * @return Failure when an owner, ID, or parent is invalid. + */ + Result<> prepareImportPublicationRecords(DataStructure& prepared, const std::vector& paths, const std::filesystem::path& sourceFile, std::vector& records, + DataObject::IdType& reservedNextId); + + /** + * @brief Runs publication with optional action-local test instrumentation. + * @param prepared Owns completed import shells. + * @param paths Lists selected paths in parent-before-child order. + * @param sourceFile Supplies diagnostic context. + * @param checkpoint Supplies the destination identity and original counter. + * @param testState Supplies private fault state, or null for ordinary publication. + * @return Publication or cleanup diagnostics. + */ + Result<> publishImportedObjectsImpl(DataStructure& prepared, const std::vector& paths, const std::filesystem::path& sourceFile, const ImportPublicationCheckpoint& checkpoint, + ImportPublicationTestState* testState); + + /** + * @brief Cleans exact placements before callbacks and conditionally restores the counter. + * @param records Keeps exact shells and removal messages alive. + * @param savedNextId Supplies the counter before reservation. + * @param reservedNextId Supplies the transaction high-water counter. + * @param testState Supplies private fault state, or null for ordinary publication. + * @param allowRefusal Enables the armed deterministic cleanup refusal. + * @return True when every transaction placement and index entry is removed. + */ + bool rollbackImportPublication(std::vector& records, DataObject::IdType savedNextId, DataObject::IdType reservedNextId, ImportPublicationTestState* testState, + bool allowRefusal); + + /** + * @brief Appends retained-placement and notification failures to an invalid Result. + * @param result Receives diagnostics after the primary publication error. + * @param records Supplies completed cleanup and notification states. + * @param sourceFile Supplies diagnostic context. + * @pre result is invalid. + */ + static void appendImportCleanupDiagnostics(Result<>& result, const std::vector& records, const std::filesystem::path& sourceFile); + + /** + * @brief Transfers failed transaction observations into action-local test storage. + * @param testState Receives observations, or null to leave records local. + * @param records Transfers ownership only when test instrumentation is active. + * @param savedNextId Supplies the original destination counter. + * @param reservedNextId Supplies the transaction high-water counter. + */ + void retainImportFailure(ImportPublicationTestState* testState, std::vector& records, DataObject::IdType savedNextId, DataObject::IdType reservedNextId); + bool insertTopLevel(const std::shared_ptr& obj); bool removeTopLevel(DataObject* data); @@ -695,4 +910,81 @@ class SIMPLNX_EXPORT DataStructure DataObject::IdType m_NextId = 1; std::shared_ptr m_FormatResolver; // Null selects the process default resolver. }; + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +/** + * @class ImportPublicationTestControl + * @brief Owns action-local faults and exact failed-publication observations. + * + * The control never exposes transaction records or destination mutation fields. + * A retained destination must remain alive until explicit cleanup completes. + * Destruction releases observations and does not retry cleanup. + */ +class SIMPLNX_EXPORT DataStructure::ImportPublicationTestControl +{ +public: + /** + * @enum Fault + * @brief Selects one deterministic publication failure. + */ + enum class Fault : uint8 + { + None, ///< Runs without injection. + BeforePublication, ///< Fails after ID reservation and before insertion. + AfterSecondInsert, ///< Fails after two complete real insertions. + AfterHierarchy, ///< Fails after the second hierarchy placement, before its parent list. + BeforeTracking, ///< Fails after the second parent list, before its weak entry. + AllocationAfterSecond, ///< Throws bad_alloc after two complete insertions. + ReturnFalse ///< Makes the second insertion fail before mutation. + }; + + /** + * @brief Creates a control with no armed fault or retained transaction. + */ + ImportPublicationTestControl(); + /** + * @brief Releases retained observations without retrying cleanup. + */ + ~ImportPublicationTestControl() noexcept; + ImportPublicationTestControl(const ImportPublicationTestControl&) = delete; + ImportPublicationTestControl(ImportPublicationTestControl&&) = delete; + ImportPublicationTestControl& operator=(const ImportPublicationTestControl&) = delete; + ImportPublicationTestControl& operator=(ImportPublicationTestControl&&) = delete; + + /** + * @brief Replaces prior observations and arms one publication fault. + * @param fault Selects the failure boundary. + * @param refuseCleanup Retains the second exact placement during automatic cleanup. + * @pre The caller excludes publication and explicit cleanup through this control. + */ + void setFault(Fault fault, bool refuseCleanup); + bool publicationFaultConsumed() const; + bool cleanupFaultConsumed() const; + std::shared_ptr retainedOwner() const; + + /** + * @brief Runs the ordinary publication operation with this control's fault state. + * @param destination Receives selected shells. + * @param prepared Owns completed import values. + * @param paths Lists selected paths in parent-before-child order. + * @param sourceFile Supplies diagnostic context. + * @param checkpoint Captures destination before preparation. + * @return The same diagnostics as publishImportedObjects, including -6217 and -6218. + * @pre The caller meets publishImportedObjects preconditions and excludes concurrent control access. + * @post Checkpoint rejection does not consume faults or replace retained observations. + */ + Result<> publish(DataStructure& destination, DataStructure& prepared, const std::vector& paths, const std::filesystem::path& sourceFile, const ImportPublicationCheckpoint& checkpoint); + + /** + * @brief Retries structural cleanup of this control's retained transaction. + * @param destination Supplies the same live destination used by the failed publication. + * @return Error -6215 when the destination differs or an exact-object conflict remains. + * @pre The caller excludes concurrent destination mutation and control access. + */ + Result<> cleanup(DataStructure& destination); + +private: + std::unique_ptr m_State; +}; +#endif } // namespace nx::core diff --git a/src/simplnx/DataStructure/EmptyDataStore.hpp b/src/simplnx/DataStructure/EmptyDataStore.hpp index 6c3fbe55ce..72183f9c25 100644 --- a/src/simplnx/DataStructure/EmptyDataStore.hpp +++ b/src/simplnx/DataStructure/EmptyDataStore.hpp @@ -93,7 +93,8 @@ class EmptyDataStore : public AbstractDataStore * @param other Source metadata store. */ EmptyDataStore(const EmptyDataStore& other) - : m_ComponentShape(other.m_ComponentShape) + : AbstractDataStore(other) + , m_ComponentShape(other.m_ComponentShape) , m_TupleShape(other.m_TupleShape) , m_NumComponents(other.m_NumComponents) , m_NumTuples(other.m_NumTuples) @@ -106,7 +107,8 @@ class EmptyDataStore : public AbstractDataStore * @param other Source metadata store. */ EmptyDataStore(EmptyDataStore&& other) noexcept - : m_ComponentShape(std::move(other.m_ComponentShape)) + : AbstractDataStore(std::move(other)) + , m_ComponentShape(std::move(other.m_ComponentShape)) , m_TupleShape(std::move(other.m_TupleShape)) , m_NumComponents(other.m_NumComponents) , m_NumTuples(other.m_NumTuples) diff --git a/src/simplnx/DataStructure/IListStore.hpp b/src/simplnx/DataStructure/IListStore.hpp index 1859d03674..e7c297524b 100644 --- a/src/simplnx/DataStructure/IListStore.hpp +++ b/src/simplnx/DataStructure/IListStore.hpp @@ -39,6 +39,15 @@ class IListStore return false; } + /** + * @brief Flushes pending list changes when the store has a backing target. + * @return Success for the default resident-store operation. + */ + [[nodiscard]] virtual Result<> flushChecked() const + { + return {}; + } + virtual usize getNumberOfTuples() const = 0; /** diff --git a/src/simplnx/DataStructure/IO/HDF5/BaseGroupIO.cpp b/src/simplnx/DataStructure/IO/HDF5/BaseGroupIO.cpp index 39b249ec64..2329fd618b 100644 --- a/src/simplnx/DataStructure/IO/HDF5/BaseGroupIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/BaseGroupIO.cpp @@ -26,19 +26,21 @@ Result<> BaseGroupIO::WriteBaseGroupData(DataStructureWriter& dataStructureWrite return result; } - return WriteDataMap(dataStructureWriter, baseGroup.getDataMap(), groupWriter, importable); + AppendWriteResult(result, WriteDataMap(dataStructureWriter, baseGroup.getDataMap(), groupWriter, importable)); + return result; } Result<> BaseGroupIO::WriteDataMap(DataStructureWriter& dataStructureWriter, const DataMap& dataMap, group_writer_type& parentGroupWriter, bool importable) { + Result<> result; for(const auto& [id, dataObject] : dataMap) { - Result<> result = dataStructureWriter.writeDataObject(dataObject.get(), parentGroupWriter); + AppendWriteResult(result, dataStructureWriter.writeDataObject(dataObject.get(), parentGroupWriter)); if(result.invalid()) { return result; } } - return {}; + return result; } } // namespace nx::core::HDF5 diff --git a/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.cpp b/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.cpp index b00b6fa0aa..0a077082ad 100644 --- a/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.cpp @@ -1,4 +1,7 @@ #include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" +#include "simplnx/DataStructure/IO/Generic/IDataStoreFormatResolver.hpp" +#include +#include #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/DataMap.hpp" @@ -25,6 +28,11 @@ Result DataStructureReader::ReadFile(const std::filesystem::path& return ReadFile(fileReader, useEmptyDataStores); } Result DataStructureReader::ReadFile(const nx::core::HDF5::FileIO& fileReader, bool useEmptyDataStores) +{ + return ReadFile(fileReader, useEmptyDataStores, {}); +} + +Result DataStructureReader::ReadFile(const nx::core::HDF5::FileIO& fileReader, bool useEmptyDataStores, std::shared_ptr resolver) { DataStructureReader dataStructureReader; auto groupReader = fileReader.openGroup(Constants::k_DataStructureTag); @@ -32,7 +40,7 @@ Result DataStructureReader::ReadFile(const nx::core::HDF5::FileIO // derived connectivity, and rectilinear-grid bounds -- as an EmptyDataStore carrying only shape and // type when useEmptyDataStores is set, keeping preflight metadata-only with no eager bulk reads from // disk. A full read (useEmptyDataStores == false) imports the real stores directly in readGroup. - return dataStructureReader.readGroup(groupReader, useEmptyDataStores); + return dataStructureReader.readGroup(groupReader, useEmptyDataStores, std::move(resolver)); } Result> DataStructureReader::ReadObject(const nx::core::HDF5::FileIO& fileReader, const DataPath& dataPath) @@ -102,6 +110,11 @@ Result<> DataStructureReader::FinishImportingObject(DataStructure& dataStructure } Result DataStructureReader::readGroup(const nx::core::HDF5::GroupIO& groupReader, bool useEmptyDataStores) +{ + return readGroup(groupReader, useEmptyDataStores, {}); +} + +Result DataStructureReader::readGroup(const nx::core::HDF5::GroupIO& groupReader, bool useEmptyDataStores, std::shared_ptr resolver) { clearDataStructure(); @@ -119,6 +132,7 @@ Result DataStructureReader::readGroup(const nx::core::HDF5::Group DataObject::IdType objectId = std::move(idResult.value()); m_CurrentStructure = DataStructure(); + m_CurrentStructure.setFormatResolver(std::move(resolver)); m_CurrentStructure.setNextId(objectId); Result<> result = HDF5::ReadDataMap(*this, m_CurrentStructure.getRootGroup(), groupReader, {}, useEmptyDataStores); if(result.invalid()) diff --git a/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp b/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp index 6e99d78b00..91d0cad379 100644 --- a/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp +++ b/src/simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp @@ -41,6 +41,16 @@ class SIMPLNX_EXPORT DataStructureReader */ static Result ReadFile(const nx::core::HDF5::FileIO& fileReader, bool useEmptyDataStores = false); + /** + * @brief Reads a structure with an explicit local storage policy. + * @param fileReader Supplies the open source file. + * @param useEmptyDataStores Selects metadata placeholders. + * + * @param resolver Supplies policy before any imported object is created; null retains ordinary policy. + * @return Imported structure and read diagnostics. + */ + static Result ReadFile(const nx::core::HDF5::FileIO& fileReader, bool useEmptyDataStores, std::shared_ptr resolver); + /** * @brief Reads a single DataObject from an HDF5 file at the specified path. * @param fileReader The HDF5 file reader to read from @@ -68,6 +78,15 @@ class SIMPLNX_EXPORT DataStructureReader */ Result readGroup(const nx::core::HDF5::GroupIO& groupReader, bool useEmptyDataStores = false); + /** + * @brief Reads a group with an explicit local storage policy. + * @param groupReader Supplies the source DataStructure group. + * @param useEmptyDataStores Selects metadata placeholders. + * @param resolver Supplies policy after reset and before object creation. + * @return Imported structure and read diagnostics. + */ + Result readGroup(const nx::core::HDF5::GroupIO& groupReader, bool useEmptyDataStores, std::shared_ptr resolver); + /** * @brief Imports a DataObject with the specified name from the target * HDF5 group. Returns any HDF5 error code that occurs. Returns 0 otherwise. diff --git a/src/simplnx/DataStructure/IO/HDF5/DataStructureWriter.cpp b/src/simplnx/DataStructure/IO/HDF5/DataStructureWriter.cpp index 23318739d9..225cc87514 100644 --- a/src/simplnx/DataStructure/IO/HDF5/DataStructureWriter.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/DataStructureWriter.cpp @@ -1,10 +1,12 @@ #include "DataStructureWriter.hpp" +#include #include "simplnx/Core/Application.hpp" #include "simplnx/DataStructure/INeighborList.hpp" #include "simplnx/DataStructure/IO/Generic/DataIOCollection.hpp" #include "simplnx/DataStructure/IO/HDF5/DataIOManager.hpp" #include "simplnx/DataStructure/IO/HDF5/IDataIO.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" #include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" @@ -153,7 +155,7 @@ Result<> DataStructureWriter::writeDataObject(const DataObject* dataObject, nx:: // without copying data. An empty override uses the normal type factory. if(auto overrideResult = Application::GetOrCreateInstance()->getIOCollection().onRecoveryWrite(*this, dataObject, parentGroup); overrideResult.has_value()) { - return overrideResult.value(); + return std::move(overrideResult.value()); } auto factory = m_IOManager->getFactoryAs(dataObject->getTypeName()); @@ -163,27 +165,22 @@ Result<> DataStructureWriter::writeDataObject(const DataObject* dataObject, nx:: return MakeErrorResult(-5, ss); } - auto result = factory->writeDataObject(*this, dataObject, parentGroup); - if(result.invalid()) - { - return result; - } - - return {}; + return factory->writeDataObject(*this, dataObject, parentGroup); } Result<> DataStructureWriter::writeDataMap(const DataMap& dataMap, nx::core::HDF5::GroupIO& parentGroup) { + Result<> result; for(const auto& [key, object] : dataMap) { - Result<> result = writeDataObject(object.get(), parentGroup); + AppendWriteResult(result, writeDataObject(object.get(), parentGroup)); if(result.invalid()) { return result; } } - return {}; + return result; } Result<> DataStructureWriter::writeDataStructure(const DataStructure& dataStructure, nx::core::HDF5::GroupIO& groupIO) diff --git a/src/simplnx/DataStructure/IO/HDF5/EdgeGeomIO.cpp b/src/simplnx/DataStructure/IO/HDF5/EdgeGeomIO.cpp index bdad24b925..e1e242d9e5 100644 --- a/src/simplnx/DataStructure/IO/HDF5/EdgeGeomIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/EdgeGeomIO.cpp @@ -1,5 +1,6 @@ #include "EdgeGeomIO.hpp" #include "DataStructureReader.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" #include "simplnx/DataStructure/Geometry/EdgeGeom.hpp" #include "simplnx/DataStructure/IO/Generic/IOConstants.hpp" @@ -56,45 +57,49 @@ Result<> EdgeGeomIO::finishImportingData(DataStructure& dataStructure, const Dat Result<> EdgeGeomIO::writeData(DataStructureWriter& dataStructureWriter, const EdgeGeom& geometry, group_writer_type& parentGroupWriter, bool importable) const { auto groupWriter = parentGroupWriter.createGroup(geometry.getName()); - INodeGeom1dIO::WriteNodeGeom1dData(dataStructureWriter, geometry, parentGroupWriter, importable); + auto result = INodeGeom1dIO::WriteNodeGeom1dData(dataStructureWriter, geometry, parentGroupWriter, importable); + if(result.invalid()) + { + return result; + } - Result<> result = WriteDataId(groupWriter, geometry.getVertexListId(), IOConstants::k_VertexListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getVertexListId(), IOConstants::k_VertexListTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getEdgeListId(), IOConstants::k_EdgeListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getEdgeListId(), IOConstants::k_EdgeListTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementContainingVertId(), IOConstants::k_ElementContainingVertTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementContainingVertId(), IOConstants::k_ElementContainingVertTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementNeighborsId(), IOConstants::k_ElementNeighborsTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementNeighborsId(), IOConstants::k_ElementNeighborsTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementCentroidsId(), IOConstants::k_ElementCentroidTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementCentroidsId(), IOConstants::k_ElementCentroidTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementSizesId(), IOConstants::k_ElementSizesTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementSizesId(), IOConstants::k_ElementSizesTag)); if(result.invalid()) { return result; } - return {}; + return result; } Result<> EdgeGeomIO::writeDataObject(DataStructureWriter& dataStructureWriter, const DataObject* dataObject, group_writer_type& parentWriter) const diff --git a/src/simplnx/DataStructure/IO/HDF5/IGridGeometryIO.cpp b/src/simplnx/DataStructure/IO/HDF5/IGridGeometryIO.cpp index 5f5089fbf1..6a25a9e55a 100644 --- a/src/simplnx/DataStructure/IO/HDF5/IGridGeometryIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/IGridGeometryIO.cpp @@ -1,4 +1,5 @@ #include "IGridGeometryIO.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" #include "DataStructureWriter.hpp" #include "simplnx/DataStructure/Geometry/IGridGeometry.hpp" @@ -46,7 +47,7 @@ Result<> IGridGeometryIO::WriteGridGeometryData(DataStructureWriter& dataStructu } auto groupWriter = parentGroup.createGroup(geometry.getName()); - Result<> writeResult = WriteDataId(groupWriter, geometry.getCellDataId(), IOConstants::k_CellDataTag); - return writeResult; + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getCellDataId(), IOConstants::k_CellDataTag)); + return result; } } // namespace nx::core::HDF5 diff --git a/src/simplnx/DataStructure/IO/HDF5/INodeGeom0dIO.cpp b/src/simplnx/DataStructure/IO/HDF5/INodeGeom0dIO.cpp index 0a5df9b1f2..b5c1d9c90d 100644 --- a/src/simplnx/DataStructure/IO/HDF5/INodeGeom0dIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/INodeGeom0dIO.cpp @@ -1,4 +1,7 @@ #include "INodeGeom0dIO.hpp" +#include "simplnx/Common/TypeTraits.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" +#include #include "DataStructureReader.hpp" #include "DataStructureWriter.hpp" @@ -71,7 +74,7 @@ Result<> INodeGeom0dIO::WriteNodeGeom0dData(DataStructureWriter& dataStructureWr DataObject::OptionalId vertexListId = geometry.getVertexListId(); - result = WriteDataId(groupWriter, vertexListId, IOConstants::k_VertexListTag); + AppendWriteResult(result, WriteDataId(groupWriter, vertexListId, IOConstants::k_VertexListTag)); if(result.invalid()) { return result; @@ -87,25 +90,31 @@ Result<> INodeGeom0dIO::WriteNodeGeom0dData(DataStructureWriter& dataStructureWr { indices[i] = i; } - result = datasetWriter.writeSpan(nx::core::HDF5::DatasetIO::DimsType{numVerts, 1}, nonstd::span{indices}); + AppendWriteResult(result, datasetWriter.writeSpan(nx::core::HDF5::DatasetIO::DimsType{numVerts, 1}, nonstd::span{indices})); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write vertex indices for geometry '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write vertex indices for geometry '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } } - result = groupWriter.writeScalarAttribute(IOConstants::k_H5_UNITS, nx::core::to_underlying(geometry.getUnits())); + AppendWriteResult(result, groupWriter.writeScalarAttribute(IOConstants::k_H5_UNITS, nx::core::to_underlying(geometry.getUnits()))); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry units for '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry units for '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = WriteDataId(groupWriter, geometry.getVertexAttributeMatrixId(), IOConstants::k_VertexDataTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getVertexAttributeMatrixId(), IOConstants::k_VertexDataTag)); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write vertex attribute matrix for geometry '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write vertex attribute matrix for geometry '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - return {}; + return result; } } // namespace nx::core::HDF5 diff --git a/src/simplnx/DataStructure/IO/HDF5/INodeGeom1dIO.cpp b/src/simplnx/DataStructure/IO/HDF5/INodeGeom1dIO.cpp index ab3f05e98a..c9537f8dfe 100644 --- a/src/simplnx/DataStructure/IO/HDF5/INodeGeom1dIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/INodeGeom1dIO.cpp @@ -1,4 +1,5 @@ #include "INodeGeom1dIO.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" #include "DataStructureReader.hpp" #include "DataStructureWriter.hpp" @@ -49,36 +50,36 @@ Result<> INodeGeom1dIO::WriteNodeGeom1dData(DataStructureWriter& dataStructureWr } auto groupWriter = parentGroupWriter.createGroup(geometry.getName()); - result = WriteDataId(groupWriter, geometry.getEdgeListId(), IOConstants::k_EdgeListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getEdgeListId(), IOConstants::k_EdgeListTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getEdgeAttributeMatrixId(), IOConstants::k_EdgeDataTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getEdgeAttributeMatrixId(), IOConstants::k_EdgeDataTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementContainingVertId(), IOConstants::k_ElementContainingVertTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementContainingVertId(), IOConstants::k_ElementContainingVertTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementNeighborsId(), IOConstants::k_ElementNeighborsTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementNeighborsId(), IOConstants::k_ElementNeighborsTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getElementCentroidsId(), IOConstants::k_ElementCentroidTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getElementCentroidsId(), IOConstants::k_ElementCentroidTag)); if(result.invalid()) { return result; } - return {}; + return result; } } // namespace nx::core::HDF5 diff --git a/src/simplnx/DataStructure/IO/HDF5/INodeGeom2dIO.cpp b/src/simplnx/DataStructure/IO/HDF5/INodeGeom2dIO.cpp index e083fd3b70..3a2719c8a1 100644 --- a/src/simplnx/DataStructure/IO/HDF5/INodeGeom2dIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/INodeGeom2dIO.cpp @@ -1,4 +1,5 @@ #include "INodeGeom2dIO.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" #include "DataStructureReader.hpp" #include "DataStructureWriter.hpp" @@ -49,24 +50,24 @@ Result<> INodeGeom2dIO::WriteNodeGeom2dData(DataStructureWriter& dataStructureWr auto groupWriter = parentGroupWriter.createGroup(geometry.getName()); - result = WriteDataId(groupWriter, geometry.getFaceListId(), IOConstants::k_FaceListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getFaceListId(), IOConstants::k_FaceListTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getFaceAttributeMatrixId(), IOConstants::k_FaceDataTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getFaceAttributeMatrixId(), IOConstants::k_FaceDataTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geometry.getUnsharedEdgesId(), IOConstants::k_UnsharedEdgeListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getUnsharedEdgesId(), IOConstants::k_UnsharedEdgeListTag)); if(result.invalid()) { return result; } - return {}; + return result; } } // namespace nx::core::HDF5 diff --git a/src/simplnx/DataStructure/IO/HDF5/INodeGeom3dIO.cpp b/src/simplnx/DataStructure/IO/HDF5/INodeGeom3dIO.cpp index cee246ff42..6a1bd335fa 100644 --- a/src/simplnx/DataStructure/IO/HDF5/INodeGeom3dIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/INodeGeom3dIO.cpp @@ -1,4 +1,5 @@ #include "INodeGeom3dIO.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" #include "DataStructureReader.hpp" #include "DataStructureWriter.hpp" @@ -49,24 +50,24 @@ Result<> INodeGeom3dIO::WriteNodeGeom3dData(DataStructureWriter& dataStructureWr auto groupWriter = parentGroup.createGroup(geom.getName()); - result = WriteDataId(groupWriter, geom.getPolyhedronListId(), IOConstants::k_PolyhedronListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geom.getPolyhedronListId(), IOConstants::k_PolyhedronListTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geom.getPolyhedraDataId(), IOConstants::k_PolyhedronDataTag); + AppendWriteResult(result, WriteDataId(groupWriter, geom.getPolyhedraDataId(), IOConstants::k_PolyhedronDataTag)); if(result.invalid()) { return result; } - result = WriteDataId(groupWriter, geom.getUnsharedFacesId(), IOConstants::k_UnsharedFaceListTag); + AppendWriteResult(result, WriteDataId(groupWriter, geom.getUnsharedFacesId(), IOConstants::k_UnsharedFaceListTag)); if(result.invalid()) { return result; } - return {}; + return result; } } // namespace nx::core::HDF5 diff --git a/src/simplnx/DataStructure/IO/HDF5/IOUtilities.hpp b/src/simplnx/DataStructure/IO/HDF5/IOUtilities.hpp index 6fab330533..1ca36cff84 100644 --- a/src/simplnx/DataStructure/IO/HDF5/IOUtilities.hpp +++ b/src/simplnx/DataStructure/IO/HDF5/IOUtilities.hpp @@ -6,6 +6,8 @@ #include "simplnx/Utilities/Parsing/HDF5/IO/GroupIO.hpp" +#include + namespace nx::core { class BaseGroup; @@ -20,6 +22,33 @@ class DataStructureWriter; class GroupIO; class ObjectIO; +/** + * @brief Appends each new write diagnostic without copying the accumulated collections. + * @param accumulated Receives diagnostics and retains failure validity, including empty error collections. + * @param next Result from the next write operation. + */ +inline void AppendWriteResult(Result<>& accumulated, Result<> next) +{ + for(auto& warning : next.warnings()) + { + accumulated.warnings().push_back(std::move(warning)); + } + if(next.invalid()) + { + if(accumulated.valid()) + { + accumulated.m_Expected = nonstd::make_unexpected(std::move(next.errors())); + } + else + { + for(auto& error : next.errors()) + { + accumulated.errors().push_back(std::move(error)); + } + } + } +} + /** * @brief Attempts to write the DataObject attributes to HDF5. * Returns a Result<> with any errors or warnings encountered during the process. diff --git a/src/simplnx/DataStructure/IO/HDF5/ImageGeomIO.cpp b/src/simplnx/DataStructure/IO/HDF5/ImageGeomIO.cpp index b1756a4b9f..36245c01d6 100644 --- a/src/simplnx/DataStructure/IO/HDF5/ImageGeomIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/ImageGeomIO.cpp @@ -1,4 +1,7 @@ #include "ImageGeomIO.hpp" +#include "simplnx/Common/TypeTraits.hpp" +#include +#include #include "DataStructureReader.hpp" #include "simplnx/Common/Array.hpp" @@ -111,29 +114,37 @@ Result<> ImageGeomIO::writeData(DataStructureWriter& dataStructureWriter, const auto groupWriter = parentGroupWriter.createGroup(geometry.getName()); - result = groupWriter.writeVectorAttribute(IOConstants::k_H5_DIMENSIONS, geometry.getDimensions().toContainer>()); + AppendWriteResult(result, groupWriter.writeVectorAttribute(IOConstants::k_H5_DIMENSIONS, geometry.getDimensions().toContainer>())); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry dimensions for '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry dimensions for '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = groupWriter.writeVectorAttribute(IOConstants::k_H5_ORIGIN, geometry.getOrigin().toContainer>()); + AppendWriteResult(result, groupWriter.writeVectorAttribute(IOConstants::k_H5_ORIGIN, geometry.getOrigin().toContainer>())); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry origin for '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry origin for '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = groupWriter.writeVectorAttribute(IOConstants::k_H5_SPACING, geometry.getSpacing().toContainer>()); + AppendWriteResult(result, groupWriter.writeVectorAttribute(IOConstants::k_H5_SPACING, geometry.getSpacing().toContainer>())); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry spacing for '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry spacing for '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = groupWriter.writeScalarAttribute(IOConstants::k_H5_UNITS, nx::core::to_underlying(geometry.getUnits())); + AppendWriteResult(result, groupWriter.writeScalarAttribute(IOConstants::k_H5_UNITS, nx::core::to_underlying(geometry.getUnits()))); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry units for '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry units for '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - return {}; + return result; } Result<> ImageGeomIO::writeDataObject(DataStructureWriter& dataStructureWriter, const DataObject* dataObject, group_writer_type& parentWriter) const diff --git a/src/simplnx/DataStructure/IO/HDF5/RectGridGeomIO.cpp b/src/simplnx/DataStructure/IO/HDF5/RectGridGeomIO.cpp index 981eef913f..f5f28b0281 100644 --- a/src/simplnx/DataStructure/IO/HDF5/RectGridGeomIO.cpp +++ b/src/simplnx/DataStructure/IO/HDF5/RectGridGeomIO.cpp @@ -1,4 +1,7 @@ #include "RectGridGeomIO.hpp" +#include "simplnx/Common/TypeTraits.hpp" +#include "simplnx/DataStructure/IO/HDF5/IOUtilities.hpp" +#include #include "DataStructureReader.hpp" #include "simplnx/DataStructure/Geometry/RectGridGeom.hpp" @@ -116,37 +119,47 @@ Result<> RectGridGeomIO::writeData(DataStructureWriter& dataStructureWriter, con dimsVector[i] = dimensions[i]; } - result = groupWriter.writeVectorAttribute(IOConstants::k_DimensionsTag, dimsVector); + AppendWriteResult(result, groupWriter.writeVectorAttribute(IOConstants::k_DimensionsTag, dimsVector)); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write dimensions for RectGridGeom '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write dimensions for RectGridGeom '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } // Write the bounds array identifiers. - result = WriteDataId(groupWriter, geometry.getXBoundsId(), IOConstants::k_XBoundsTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getXBoundsId(), IOConstants::k_XBoundsTag)); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write X bounds for RectGridGeom '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write X bounds for RectGridGeom '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = WriteDataId(groupWriter, geometry.getYBoundsId(), IOConstants::k_YBoundsTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getYBoundsId(), IOConstants::k_YBoundsTag)); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write Y bounds for RectGridGeom '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write Y bounds for RectGridGeom '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = WriteDataId(groupWriter, geometry.getZBoundsId(), IOConstants::k_ZBoundsTag); + AppendWriteResult(result, WriteDataId(groupWriter, geometry.getZBoundsId(), IOConstants::k_ZBoundsTag)); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write Z bounds for RectGridGeom '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write Z bounds for RectGridGeom '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - result = groupWriter.writeScalarAttribute(IOConstants::k_H5_UNITS, nx::core::to_underlying(geometry.getUnits())); + AppendWriteResult(result, groupWriter.writeScalarAttribute(IOConstants::k_H5_UNITS, nx::core::to_underlying(geometry.getUnits()))); if(result.invalid()) { - return MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry units for RectGridGeom '{}'", geometry.getName())); + auto errorResult = MakeErrorResult(result.errors()[0].code, fmt::format("Failed to write geometry units for RectGridGeom '{}'", geometry.getName())); + errorResult.warnings() = std::move(result.warnings()); + return errorResult; } - return {}; + return result; } Result<> RectGridGeomIO::writeDataObject(DataStructureWriter& dataStructureWriter, const DataObject* dataObject, group_writer_type& parentWriter) const diff --git a/src/simplnx/DataStructure/NeighborList.cpp b/src/simplnx/DataStructure/NeighborList.cpp index 3b949f46cd..202fee74b4 100644 --- a/src/simplnx/DataStructure/NeighborList.cpp +++ b/src/simplnx/DataStructure/NeighborList.cpp @@ -456,6 +456,21 @@ const IListStore* NeighborList::getIListStore() const return m_Store.get(); } +template +Result<> NeighborList::flushChecked() const +{ + if(m_Store == nullptr) + { + return MakeErrorResult(-6070, fmt::format("Cannot flush neighbor list '{}': its list store is null.", getName())); + } + auto result = m_Store->flushChecked(); + if(result.invalid() && result.errors().empty()) + { + result.errors().push_back({-6070, fmt::format("Cannot flush neighbor list '{}': its list store failed without an error diagnostic.", getName())}); + } + return result; +} + template std::shared_ptr::store_type> NeighborList::getStore() const { diff --git a/src/simplnx/DataStructure/NeighborList.hpp b/src/simplnx/DataStructure/NeighborList.hpp index 5a46ca108f..2c5f15cf64 100644 --- a/src/simplnx/DataStructure/NeighborList.hpp +++ b/src/simplnx/DataStructure/NeighborList.hpp @@ -396,6 +396,12 @@ class NeighborList : public INeighborList */ const IListStore* getIListStore() const override; + /** + * @brief Flushes the list store and preserves its diagnostics. + * @return Store diagnostics, or error -6070 for missing storage or a failure without an error diagnostic. + */ + [[nodiscard]] Result<> flushChecked() const override; + /** * @brief Returns a shared_ptr to the underlying list store. * @return std::shared_ptr diff --git a/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.cpp b/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.cpp index c74666eb81..040427df9d 100644 --- a/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.cpp +++ b/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.cpp @@ -1,14 +1,22 @@ #include "ImportH5ObjectPathsAction.hpp" +#include "simplnx/Common/Result.hpp" #include "simplnx/Common/StringLiteralFormatting.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/DataStructure/BaseGroup.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataObject.hpp" #include "simplnx/DataStructure/EmptyDataStore.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" #include "simplnx/DataStructure/Geometry/IGridGeometry.hpp" +#include "simplnx/DataStructure/Geometry/INodeGeometry0D.hpp" +#include "simplnx/DataStructure/Geometry/INodeGeometry1D.hpp" +#include "simplnx/DataStructure/Geometry/INodeGeometry2D.hpp" #include "simplnx/DataStructure/Geometry/INodeGeometry3D.hpp" #include "simplnx/DataStructure/Geometry/RectGridGeom.hpp" #include "simplnx/DataStructure/IDataArray.hpp" +#include "simplnx/DataStructure/INeighborList.hpp" +#include "simplnx/DataStructure/StringArray.hpp" #include "simplnx/Utilities/ArrayCreationUtilities.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" @@ -16,12 +24,22 @@ #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp" #include "simplnx/Utilities/StoreCopyUtilities.hpp" -#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include using namespace nx::core; namespace fs = std::filesystem; @@ -29,7 +47,139 @@ namespace fs = std::filesystem; namespace { /** - * @struct SetImportedPlaceholderFormatFunctor + * @struct PlacementPlan + * @brief Records each selected path's source owner and reserved destination ID. + */ +struct PlacementPlan +{ + DataPath Path; + std::shared_ptr SourcePtr; + DataObject::IdType SourceId = 0; + DataObject::IdType AssignedId = 0; + bool ExtraPlacement = false; +}; + +/** + * @struct PlacementPathLess + * @brief Orders path components so all descendants of one path form a contiguous range. + */ +struct PlacementPathLess +{ + bool operator()(const DataPath& left, const DataPath& right) const + { + const auto commonLength = std::min(left.getLength(), right.getLength()); + for(usize component = 0; component < commonLength; ++component) + { + if(left[component] != right[component]) + { + return left[component] < right[component]; + } + } + return left.getLength() < right.getLength(); + } +}; + +bool IsStrictDescendant(const DataPath& candidate, const DataPath& parent) +{ + if(candidate.getLength() <= parent.getLength()) + { + return false; + } + for(usize component = 0; component < parent.getLength(); ++component) + { + if(candidate[component] != parent[component]) + { + return false; + } + } + return true; +} + +/** + * @class GeometryReferenceRemapper + * @brief Maps geometry references through selected placement paths without reading array values. + * + * The finalized placement vector must outlive this index and remain unchanged. + */ +class GeometryReferenceRemapper +{ +public: + /** + * @brief Indexes the complete selection after all placement IDs are assigned. + * @param placements Supplies stable entries that must outlive this index. + */ + explicit GeometryReferenceRemapper(const std::vector& placements) + { + m_PlacementsBySourceId.reserve(placements.size()); + for(const auto& placement : placements) + { + m_PlacementsByPath.emplace(placement.Path, &placement); + m_PlacementsBySourceId[placement.SourceId].push_back(&placement); + } + for(auto& sourceEntry : m_PlacementsBySourceId) + { + auto& candidates = sourceEntry.second; + std::ranges::sort(candidates, PlacementPathLess{}, &PlacementPlan::Path); + } + } + + ~GeometryReferenceRemapper() noexcept = default; + + GeometryReferenceRemapper(const GeometryReferenceRemapper&) = delete; + GeometryReferenceRemapper(GeometryReferenceRemapper&&) = delete; + GeometryReferenceRemapper& operator=(const GeometryReferenceRemapper&) = delete; + GeometryReferenceRemapper& operator=(GeometryReferenceRemapper&&) = delete; + + /** + * @brief Selects a local dependency without changing the external or ambiguous-reference policy. + * @param geometryPath Identifies the exact geometry placement. + * @param sourceId Supplies a retained reference after source-wide ID reassignment. + * @return The direct-child or unique descendant ID, or the unchanged reference when neither rule selects a target. + */ + DataObject::OptionalId mapReference(const DataPath& geometryPath, DataObject::OptionalId sourceId) const + { + if(!sourceId.has_value()) + { + return sourceId; + } + const auto candidates = m_PlacementsBySourceId.find(*sourceId); + if(candidates == m_PlacementsBySourceId.end()) + { + return sourceId; + } + const auto direct = m_PlacementsByPath.find(geometryPath.createChildPath(candidates->second.front()->SourcePtr->getName())); + if(direct != m_PlacementsByPath.end() && direct->second->SourceId == *sourceId) + { + return direct->second->AssignedId; + } + const auto& sourcePlacements = candidates->second; + auto first = std::ranges::lower_bound(sourcePlacements, geometryPath, PlacementPathLess{}, &PlacementPlan::Path); + if(first != sourcePlacements.end() && (*first)->Path == geometryPath) + { + ++first; + } + if(first == sourcePlacements.end() || !IsStrictDescendant((*first)->Path, geometryPath)) + { + // A selected external target can be shared when this geometry has no selected local copy. + return sourceId; + } + // Component ordering keeps descendants contiguous, so two candidates establish ambiguity without a scan. + const auto second = std::next(first); + if(second != sourcePlacements.end() && IsStrictDescendant((*second)->Path, geometryPath)) + { + // An ID does not identify a preferred alias. Keep the canonical reference when local paths are ambiguous. + return sourceId; + } + return (*first)->AssignedId; + } + +private: + std::map m_PlacementsByPath; + std::unordered_map> m_PlacementsBySourceId; +}; + +/** + * @struct PlanImportedArrayStoreFunctor * @brief Replaces an imported array store with a resolver-planned placeholder. */ struct PlanImportedArrayStoreFunctor @@ -40,21 +190,24 @@ struct PlanImportedArrayStoreFunctor * @param destination Provides the resolver and the context it judges. * @param targetPath Identifies where the array will live. * @param dataArray Receives the planned store. + * @param allowPopulated Permits only eager values from the legacy Statistics contract. * @return Resolution or store-replacement failures. * - * The plan is resolved against the destination because that is where the array will live. A - * placeholder receives a planned placeholder; an array the cache already materialized is copied - * into the destination's selected format so both routes honor the same decision. + * All selected shells are present before policy runs. Ordinary arrays remain placeholders; legacy Statistics values receive an independent selected-format copy. */ template - Result<> operator()(DataStructure& destination, const DataPath& targetPath, IDataArray* dataArray) const + Result<> operator()(DataStructure& destination, const DataPath& targetPath, IDataArray* dataArray, bool allowPopulated) const { auto* typedArray = dynamic_cast*>(dataArray); if(typedArray == nullptr || typedArray->getIDataStore() == nullptr) { - return {}; + return MakeErrorResult(-6207, fmt::format("Cannot plan imported numeric array '{}': its typed array or store is unavailable.", targetPath.toString())); } const IDataStore* sourceStore = typedArray->getIDataStore(); + if(sourceStore->getStoreType() != IDataStore::StoreType::Empty && !allowPopulated) + { + return MakeErrorResult(-6208, fmt::format("Cannot plan imported numeric array '{}': ordinary source values were materialized before destination planning.", targetPath.toString())); + } Result formatResult; try @@ -85,6 +238,7 @@ struct PlanImportedArrayStoreFunctor result.warnings().insert(result.warnings().begin(), std::make_move_iterator(warnings.begin()), std::make_move_iterator(warnings.end())); return result; } + warnings.insert(warnings.end(), std::make_move_iterator(plannedResult.warnings().begin()), std::make_move_iterator(plannedResult.warnings().end())); replaceResult = typedArray->setDataStore(std::shared_ptr>(std::move(plannedResult.value()))); } else @@ -114,15 +268,62 @@ struct PlanImportedArrayStoreFunctor } }; -Result<> applyPreflightStoragePlan(DataStructure& dataStructure, const DataPath& targetPath, IDataArray& dataArray) +Result<> applyPreflightStoragePlan(DataStructure& dataStructure, const DataPath& targetPath, IDataArray& dataArray, bool allowPopulated) { - return ExecuteDataFunction(PlanImportedArrayStoreFunctor{}, dataArray.getDataType(), dataStructure, targetPath, &dataArray); + return ExecuteDataFunction(PlanImportedArrayStoreFunctor{}, dataArray.getDataType(), dataStructure, targetPath, &dataArray, allowPopulated); } + +/** + * @brief Applies one ID transformation to every serialized geometry reference. + * @tparam Transform Maps an optional source ID to its retained or replacement ID. + * @param geometry Receives the transformed references. + * @param transform Preserves, clears, or replaces each reference without reading array values. + */ +template +void TransformGeometryReferences(IGeometry& geometry, const Transform& transform) +{ + geometry.setElementSizesId(transform(geometry.getElementSizesId())); + if(auto* gridPtr = dynamic_cast(&geometry); gridPtr != nullptr) + { + gridPtr->setCellData(transform(gridPtr->getCellDataId())); + } + if(auto* nodePtr = dynamic_cast(&geometry); nodePtr != nullptr) + { + nodePtr->setVertexListId(transform(nodePtr->getVertexListId())); + nodePtr->setVertexDataId(transform(nodePtr->getVertexAttributeMatrixId())); + } + if(auto* nodePtr = dynamic_cast(&geometry); nodePtr != nullptr) + { + nodePtr->setEdgeListId(transform(nodePtr->getEdgeListId())); + nodePtr->setEdgeDataId(transform(nodePtr->getEdgeAttributeMatrixId())); + nodePtr->setElementContainingVertId(transform(nodePtr->getElementContainingVertId())); + nodePtr->setElementNeighborsId(transform(nodePtr->getElementNeighborsId())); + nodePtr->setElementCentroidsId(transform(nodePtr->getElementCentroidsId())); + } + if(auto* nodePtr = dynamic_cast(&geometry); nodePtr != nullptr) + { + nodePtr->setFaceListId(transform(nodePtr->getFaceListId())); + nodePtr->setFaceDataId(transform(nodePtr->getFaceAttributeMatrixId())); + nodePtr->setUnsharedEdgesId(transform(nodePtr->getUnsharedEdgesId())); + } + if(auto* nodePtr = dynamic_cast(&geometry); nodePtr != nullptr) + { + nodePtr->setPolyhedronListId(transform(nodePtr->getPolyhedronListId())); + nodePtr->setPolyhedraDataId(transform(nodePtr->getPolyhedraAttributeMatrixId())); + nodePtr->setUnsharedFacedId(transform(nodePtr->getUnsharedFacesId())); + } + if(auto* rectGridPtr = dynamic_cast(&geometry); rectGridPtr != nullptr) + { + rectGridPtr->setXBoundsId(transform(rectGridPtr->getXBoundsId())); + rectGridPtr->setYBoundsId(transform(rectGridPtr->getYBoundsId())); + rectGridPtr->setZBoundsId(transform(rectGridPtr->getZBoundsId())); + } +} + /** * @brief Clears geometry references to objects outside the exact import selection. * @param sourceStructure Owns the imported object copies. - * @param paths Specifies the objects that the action - * inserts. + * @param paths Specifies the objects that the action inserts. * * Metadata and selected loaders can retain unselected objects. Their IDs must not survive renumbering as geometry references. */ @@ -146,48 +347,27 @@ void clearExcludedGeometryReferences(DataStructure& sourceStructure, const std:: { continue; } - geometry->setElementSizesId(selectedReference(geometry->getElementSizesId())); - if(auto* grid = dynamic_cast(geometry); grid != nullptr) - { - grid->setCellData(selectedReference(grid->getCellDataId())); - } - if(auto* node = dynamic_cast(geometry); node != nullptr) - { - node->setVertexListId(selectedReference(node->getVertexListId())); - node->setVertexDataId(selectedReference(node->getVertexAttributeMatrixId())); - } - if(auto* node = dynamic_cast(geometry); node != nullptr) - { - node->setEdgeListId(selectedReference(node->getEdgeListId())); - node->setEdgeDataId(selectedReference(node->getEdgeAttributeMatrixId())); - node->setElementContainingVertId(selectedReference(node->getElementContainingVertId())); - node->setElementNeighborsId(selectedReference(node->getElementNeighborsId())); - node->setElementCentroidsId(selectedReference(node->getElementCentroidsId())); - } - if(auto* node = dynamic_cast(geometry); node != nullptr) - { - node->setFaceListId(selectedReference(node->getFaceListId())); - node->setFaceDataId(selectedReference(node->getFaceAttributeMatrixId())); - node->setUnsharedEdgesId(selectedReference(node->getUnsharedEdgesId())); - } - if(auto* node = dynamic_cast(geometry); node != nullptr) - { - node->setPolyhedronListId(selectedReference(node->getPolyhedronListId())); - node->setPolyhedraDataId(selectedReference(node->getPolyhedraAttributeMatrixId())); - node->setUnsharedFacedId(selectedReference(node->getUnsharedFacesId())); - } - if(auto* rectGrid = dynamic_cast(geometry); rectGrid != nullptr) - { - rectGrid->setXBoundsId(selectedReference(rectGrid->getXBoundsId())); - rectGrid->setYBoundsId(selectedReference(rectGrid->getYBoundsId())); - rectGrid->setZBoundsId(selectedReference(rectGrid->getZBoundsId())); - } + TransformGeometryReferences(*geometry, selectedReference); } } } // namespace namespace nx::core { +struct ImportH5ObjectPathsAction::TestState +{ + TestState() = default; + ~TestState() noexcept = default; + TestState(const TestState&) = delete; + TestState(TestState&&) = delete; + TestState& operator=(const TestState&) = delete; + TestState& operator=(TestState&&) = delete; + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + DataStructure::ImportPublicationTestControl Control; +#endif +}; + ImportH5ObjectPathsAction::ImportH5ObjectPathsAction(const fs::path& importFile, const PathsType& paths) : IDataCreationAction(DataPath{}) , m_H5FilePath(importFile) @@ -200,100 +380,217 @@ ImportH5ObjectPathsAction::~ImportH5ObjectPathsAction() noexcept = default; Result<> ImportH5ObjectPathsAction::apply(DataStructure& dataStructure, Mode mode) const { - static constexpr StringLiteral prefix = "ImportH5ObjectPathsAction: "; - - // Execute materializes selected objects before the merge. Preflight uses cached metadata. - auto result = DREAM3D::Dream3dPreflightCache::Instance().fetch(m_H5FilePath); - if(result.invalid()) + auto metadata = DREAM3D::Dream3dPreflightCache::Instance().fetchNeutralMetadata(m_H5FilePath); + if(metadata.invalid()) + { + return ConvertResult(std::move(metadata)); + } + const bool legacy = metadata.value().fileVersion == DREAM3D::k_LegacyFileVersion; + DataStructure sourceStructure = std::move(metadata.value().dataStructure); + Result<> importResult = ConvertResult(std::move(metadata)); + if(m_Paths.empty()) { - return ConvertResult(std::move(result)); + return importResult; } - if(mode == Mode::Execute) + std::set selectedPaths; + for(const auto& path : m_Paths) { - std::vector selectedPaths; - selectedPaths.reserve(m_Paths.size()); - for(const auto& path : m_Paths) + if(dataStructure.containsData(path)) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6203, fmt::format("Cannot import '{}' from '{}': the destination path already exists. Rename it or exclude this path.", + path.toString(), m_H5FilePath.string()))); + } + if(sourceStructure.containsData(path) && !selectedPaths.insert(path).second) { - if(result.value().containsData(path)) + return MergeResults(std::move(importResult), + MakeErrorResult(-6203, fmt::format("Cannot import '{}' from '{}': the same path is selected more than once.", path.toString(), m_H5FilePath.string()))); + } + } + if(selectedPaths.empty()) + { + return importResult; + } + clearExcludedGeometryReferences(sourceStructure, m_Paths); + const DataStructure::ImportPublicationCheckpoint checkpoint(dataStructure); + const auto savedNextId = checkpoint.nextId(); + const auto startId = std::max(savedNextId, DataObject::IdType{1}); + constexpr auto maximumId = std::numeric_limits::max(); + const auto sourceCount = sourceStructure.getLiveObjectCount(); + if(sourceCount > maximumId - startId) + { + return MergeResults(std::move(importResult), + MakeErrorResult(-6213, fmt::format("Cannot import '{}': source identifiers starting at {} exceed the maximum next identifier {}.", m_H5FilePath.string(), startId, maximumId))); + } + DataObject::IdType extraPlacementCount = 0; + std::unordered_set countedSources; + for(const auto& path : selectedPaths) + { + const auto* objectPtr = sourceStructure.getData(path); + if(objectPtr == nullptr) + { + continue; + } + if(!countedSources.insert(objectPtr).second) + { + if(extraPlacementCount == maximumId - startId - sourceCount) { - selectedPaths.push_back(path); + return MergeResults( + std::move(importResult), + MakeErrorResult(-6213, fmt::format("Cannot import '{}': additional placement count {} does not fit after {} source identifiers starting at {} (maximum next identifier {}).", + m_H5FilePath.string(), extraPlacementCount + 1, sourceCount, startId, maximumId))); } + ++extraPlacementCount; } - result = DREAM3D::LoadDataStructureArrays(m_H5FilePath, selectedPaths); } - if(result.invalid()) + sourceStructure.resetIds(startId); + const auto sourceNextId = sourceStructure.getNextId(); + + std::vector placements; + placements.reserve(selectedPaths.size()); { - return ConvertResult(std::move(result)); + const auto occupiedIds = dataStructure.getOccupiedObjectIds(); + + auto sortedPaths = m_Paths; + std::sort(sortedPaths.begin(), sortedPaths.end(), [](const DataPath& a, const DataPath& b) { return a.getLength() < b.getLength(); }); + std::unordered_set plannedSources; + std::set assignedIds; + auto nextCloneId = sourceNextId; + // Extra copies use IDs beyond every source object, including source objects staged later or omitted from this selection. + for(const auto& path : sortedPaths) + { + auto sourceObject = sourceStructure.getSharedData(path); + if(sourceObject == nullptr) + { + continue; + } + const auto sourceId = sourceObject->getId(); + if(sourceStructure.getData(path) != sourceObject.get()) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6213, fmt::format("Cannot import '{}' from '{}': its source path and identifier {} do not name the same object.", path.toString(), + m_H5FilePath.string(), sourceId))); + } + if(sourceId < startId || sourceId >= sourceNextId || sourceId == maximumId || occupiedIds.contains(sourceId)) + { + return MergeResults(std::move(importResult), + MakeErrorResult(-6213, fmt::format("Cannot import '{}' from '{}': source identifier {} conflicts with the destination or its checked source range.", path.toString(), + m_H5FilePath.string(), sourceId))); + } + const bool extraPlacement = !plannedSources.insert(sourceObject.get()).second; + auto assignedId = sourceId; + if(extraPlacement) + { + if(nextCloneId == maximumId) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6213, fmt::format("Cannot copy placement '{}' from '{}': identifier {} leaves no representable next identifier.", + path.toString(), m_H5FilePath.string(), nextCloneId))); + } + assignedId = nextCloneId++; + } + if(occupiedIds.contains(assignedId) || !assignedIds.insert(assignedId).second) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6213, fmt::format("Cannot import '{}' from '{}': identifier {} conflicts with an existing or selected object.", path.toString(), + m_H5FilePath.string(), assignedId))); + } + placements.push_back({.Path = path, .SourcePtr = std::move(sourceObject), .SourceId = sourceId, .AssignedId = assignedId, .ExtraPlacement = extraPlacement}); + } } - DataStructure sourceStructure = std::move(result.value()); - Result<> importResult = ConvertResult(std::move(result)); - clearExcludedGeometryReferences(sourceStructure, m_Paths); - // Renumber source objects before merge to avoid collisions with pipeline objects. - sourceStructure.resetIds(dataStructure.getNextId()); + const GeometryReferenceRemapper referenceRemapper(placements); - // Stage the whole import against a copy of the destination. Planning needs the destination's real - // context, and a failure part way through must leave the caller's structure untouched, so nothing - // reaches dataStructure until every selected path has been planned and materialized. DataStructure preparedStructure = dataStructure; - - // Insert parents before children because every child requires its parent path. - auto sortedPaths = m_Paths; - std::sort(sortedPaths.begin(), sortedPaths.end(), [](const DataPath& a, const DataPath& b) { return a.getLength() < b.getLength(); }); - + if(extraPlacementCount != 0) + { + preparedStructure.setNextId(sourceNextId); + } + std::unordered_map> firstStagedCopies; std::vector preparedPaths; - std::vector importedLeafPaths; - for(const auto& targetPath : sortedPaths) + preparedPaths.reserve(placements.size()); + for(const auto& placement : placements) { - // preparedStructure starts as a copy of the destination, so this catches both objects that were - // already there and objects staged earlier in this same import. - if(preparedStructure.getDataAs(targetPath) != nullptr) + const auto& path = placement.Path; + if(preparedStructure.containsData(path)) { - return MergeResults(std::move(importResult), MakeErrorResult(-6203, fmt::format("{}Unable to import DataObject at '{}' because an object " - "already exists at that path. Consider renaming the existing object before importing, or " - "exclude this path from the import selection.", - prefix, targetPath.toString()))); + return MergeResults(std::move(importResult), MakeErrorResult(-6203, fmt::format("Cannot import '{}' from '{}': the destination path already exists. Rename it or exclude this path.", + path.toString(), m_H5FilePath.string()))); } - - if(!sourceStructure.containsData(targetPath)) + auto* collisionPtr = preparedStructure.getData(placement.SourceId); + if(placement.ExtraPlacement) { - continue; - } - - // The loaded source owns resolver-selected stores. Copy group shells without - // children because selected child paths insert independently. - const auto sourceObject = sourceStructure.getSharedData(targetPath); - const auto objectCopy = std::shared_ptr(sourceObject->shallowCopy()); - // Both modes plan storage, so the resolver sees every imported numeric array and decides where it - // lives before the array is inserted. Execute additionally fills the planned placeholders below. - if(auto* dataArray = dynamic_cast(objectCopy.get()); dataArray != nullptr) - { - importResult = MergeResults(std::move(importResult), applyPreflightStoragePlan(preparedStructure, targetPath, *dataArray)); - if(importResult.invalid()) + const auto first = firstStagedCopies.find(placement.SourcePtr.get()); + if(first == firstStagedCopies.end() || collisionPtr != first->second.get() || first->second->getId() != placement.SourceId || preparedStructure.getNextId() != placement.AssignedId) { - return importResult; + return MergeResults(std::move(importResult), + MakeErrorResult(-6213, fmt::format("Cannot stage alias '{}' from '{}': source identifier {} or prepared next identifier {} does not match planned clone {}.", + path.toString(), m_H5FilePath.string(), placement.SourceId, preparedStructure.getNextId(), placement.AssignedId))); } - importedLeafPaths.push_back(targetPath); } - if(const auto group = std::dynamic_pointer_cast(objectCopy); group != nullptr) + else if(collisionPtr != nullptr) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6213, fmt::format("Cannot stage '{}' from '{}': source identifier {} already belongs to another prepared object.", path.toString(), + m_H5FilePath.string(), placement.SourceId))); + } + auto shell = DataStructure::makeImportPublicationCopy(*placement.SourcePtr); + if(shell == nullptr) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6202, fmt::format("Cannot prepare an independent copy of '{}' from '{}'.", path.toString(), m_H5FilePath.string()))); + } + if(shell->getId() != placement.SourceId || shell.get() == collisionPtr) + { + return MergeResults(std::move(importResult), + MakeErrorResult(-6213, fmt::format("Cannot stage '{}' from '{}': detached copy identifier {} must equal source identifier {} and its owner must be independent.", + path.toString(), m_H5FilePath.string(), shell->getId(), placement.SourceId))); + } + if(auto* geometryPtr = dynamic_cast(shell.get()); geometryPtr != nullptr) + { + TransformGeometryReferences(*geometryPtr, [&](DataObject::OptionalId referenceId) { return referenceRemapper.mapReference(path, referenceId); }); + } + // Only an exact known first copy can trigger insert's one-ID assignment in this unpublished structure. + if(!preparedStructure.insert(shell, path.getParent())) + { + return MergeResults(std::move(importResult), MakeErrorResult(-6202, fmt::format("Cannot stage '{}' from '{}': parent '{}' is missing or rejects the object.", path.toString(), + m_H5FilePath.string(), path.getParent().toString()))); + } + if(shell->getId() != placement.AssignedId || preparedStructure.getData(path) != shell.get() || preparedStructure.getData(placement.AssignedId) != shell.get()) { - group->clear(); + return MergeResults(std::move(importResult), MakeErrorResult(-6213, fmt::format("Cannot stage '{}' from '{}': assigned identifier {} or its lookups do not match planned identifier {}.", + path.toString(), m_H5FilePath.string(), shell->getId(), placement.AssignedId))); } - if(!preparedStructure.insert(objectCopy, targetPath.getParent())) + if(!placement.ExtraPlacement) { - return MergeResults(std::move(importResult), MakeErrorResult(-6202, fmt::format("{}Unable to insert DataObject at path '{}' into the DataStructure. " - "The parent path '{}' may not exist.", - prefix, targetPath.toString(), targetPath.getParent().toString()))); + firstStagedCopies.emplace(placement.SourcePtr.get(), shell); } - preparedPaths.push_back(targetPath); + preparedPaths.push_back(path); } + std::vector importedLeafPaths; + importedLeafPaths.reserve(preparedPaths.size()); + // Policy sees all selected shells and the original destination context before ordinary values are read. + for(const auto& path : preparedPaths) + { + auto* objectPtr = preparedStructure.getData(path); + if(auto* arrayPtr = dynamic_cast(objectPtr); arrayPtr != nullptr) + { + const bool eagerStatistics = legacy && path.getLength() >= 2 && path[1] == "Statistics"; + auto planResult = applyPreflightStoragePlan(preparedStructure, path, *arrayPtr, eagerStatistics); + if(planResult.invalid()) + { + return MergeResults(std::move(importResult), std::move(planResult)); + } + auto& warnings = planResult.warnings(); + importResult.warnings().insert(importResult.warnings().end(), std::make_move_iterator(warnings.begin()), std::make_move_iterator(warnings.end())); + importedLeafPaths.push_back(path); + } + else if(dynamic_cast(objectPtr) != nullptr || dynamic_cast(objectPtr) != nullptr) + { + importedLeafPaths.push_back(path); + } + } if(mode == Mode::Execute && !importedLeafPaths.empty()) { auto fileReader = HDF5::FileIO::ReadFile(m_H5FilePath); if(!fileReader.isValid()) { - return MergeResults(std::move(importResult), MakeErrorResult(-6206, fmt::format("{}Unable to open '{}' for materialization.", prefix, m_H5FilePath.string()))); + return MergeResults(std::move(importResult), MakeErrorResult(-6206, fmt::format("Cannot open '{}' to materialize the selected import.", m_H5FilePath.string()))); } importResult = MergeResults(std::move(importResult), DREAM3D::detail::MaterializeImportedPaths(preparedStructure, fileReader, importedLeafPaths)); if(importResult.invalid()) @@ -302,30 +599,50 @@ Result<> ImportH5ObjectPathsAction::apply(DataStructure& dataStructure, Mode mod } } - // Every selected path planned and materialized, so publish the staged objects. - for(const auto& targetPath : preparedPaths) +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + auto publicationResult = m_TestState == nullptr ? dataStructure.publishImportedObjects(preparedStructure, preparedPaths, m_H5FilePath, checkpoint) : + m_TestState->Control.publish(dataStructure, preparedStructure, preparedPaths, m_H5FilePath, checkpoint); +#else + auto publicationResult = dataStructure.publishImportedObjects(preparedStructure, preparedPaths, m_H5FilePath, checkpoint); +#endif + // Publication produces no warnings. Transfer prior warnings without allocating after destination mutation. + publicationResult.warnings().swap(importResult.warnings()); + return publicationResult; +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +void ImportH5ObjectPathsAction::setPublicationFaultForTesting(PublicationFaultType fault, bool refuseCleanup) const +{ + m_TestState = std::make_shared(); + m_TestState->Control.setFault(fault, refuseCleanup); +} + +bool ImportH5ObjectPathsAction::publicationFaultConsumedForTesting() const +{ + return m_TestState != nullptr && m_TestState->Control.publicationFaultConsumed(); +} + +bool ImportH5ObjectPathsAction::cleanupFaultConsumedForTesting() const +{ + return m_TestState != nullptr && m_TestState->Control.cleanupFaultConsumed(); +} + +std::shared_ptr ImportH5ObjectPathsAction::retainedOwnerForTesting() const +{ + return m_TestState == nullptr ? nullptr : m_TestState->Control.retainedOwner(); +} + +Result<> ImportH5ObjectPathsAction::cleanupFailedImportForTesting(DataStructure& destination) const +{ + if(m_TestState == nullptr) { - const auto preparedObject = preparedStructure.getSharedData(targetPath); - if(preparedObject == nullptr) - { - continue; - } - const auto objectCopy = std::shared_ptr(preparedObject->shallowCopy()); - if(const auto group = std::dynamic_pointer_cast(objectCopy); group != nullptr) - { - group->clear(); - } - if(!dataStructure.insert(objectCopy, targetPath.getParent())) - { - return MergeResults(std::move(importResult), MakeErrorResult(-6202, fmt::format("{}Unable to insert DataObject at path '{}' into the DataStructure. " - "The parent path '{}' may not exist.", - prefix, targetPath.toString(), targetPath.getParent().toString()))); - } + return MakeErrorResult(-6215, "The selected import has no retained rollback records for this destination."); } - - return importResult; + return m_TestState->Control.cleanup(destination); } +#endif + IDataAction::UniquePointer ImportH5ObjectPathsAction::clone() const { return std::make_unique(m_H5FilePath, m_Paths); diff --git a/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp b/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp index 923eb474fc..74dbf3a9d7 100644 --- a/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp +++ b/src/simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp @@ -1,6 +1,7 @@ #pragma once #include "simplnx/DataStructure/DataPath.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" #include "simplnx/Filter/Output.hpp" #include @@ -37,11 +38,10 @@ class SIMPLNX_EXPORT ImportH5ObjectPathsAction : public IDataCreationAction ImportH5ObjectPathsAction& operator=(ImportH5ObjectPathsAction&&) noexcept = delete; /** - * @brief Applies this action's change to the given DataStructure in the given mode. - * Returns any warnings/errors. On error, DataStructure is not guaranteed to be consistent. + * @brief Plans selected metadata, materializes requested values, and publishes completed objects. * @param dataStructure The DataStructure to modify * @param mode The mode (Preflight or Execute) - * @return Result<> Result with any errors or warnings + * @return Import diagnostics, including retained paths when exact rollback cannot complete. */ Result<> apply(DataStructure& dataStructure, Mode mode) const override; @@ -57,8 +57,34 @@ class SIMPLNX_EXPORT ImportH5ObjectPathsAction : public IDataCreationAction */ std::vector getAllCreatedPaths() const override; +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + /** + * @brief Selects an action-local failure in the destination-owned publication operation. + */ + using PublicationFaultType = DataStructure::ImportPublicationTestControl::Fault; + + /** + * @brief Arms one action-local publication fault. + * @param fault Selects the failure boundary. + * @param refuseCleanup Retains the second object's exact placement during automatic cleanup. + */ + void setPublicationFaultForTesting(PublicationFaultType fault, bool refuseCleanup = false) const; + bool publicationFaultConsumedForTesting() const; + bool cleanupFaultConsumedForTesting() const; + std::shared_ptr retainedOwnerForTesting() const; + + /** + * @brief Completes a deliberately refused rollback through the same private cleanup route. + * @param destination Contains the retained transaction objects. + * @return Cleanup diagnostics. + */ + Result<> cleanupFailedImportForTesting(DataStructure& destination) const; +#endif + private: + struct TestState; std::filesystem::path m_H5FilePath; PathsType m_Paths; + mutable std::shared_ptr m_TestState; }; } // namespace nx::core diff --git a/src/simplnx/Parameters/ArraySelectionParameter.cpp b/src/simplnx/Parameters/ArraySelectionParameter.cpp index bcd8805ac7..2c3fb37594 100644 --- a/src/simplnx/Parameters/ArraySelectionParameter.cpp +++ b/src/simplnx/Parameters/ArraySelectionParameter.cpp @@ -1,4 +1,7 @@ #include "ArraySelectionParameter.hpp" +#include "simplnx/DataStructure/IDataStore.hpp" +#include "simplnx/Filter/IParameter.hpp" +#include #include "simplnx/Common/Any.hpp" #include "simplnx/Common/StringLiteralFormatting.hpp" @@ -93,7 +96,7 @@ Result ArraySelectionParameter::fromJsonImpl(const nlohmann::json& jso IParameter::UniquePointer ArraySelectionParameter::clone() const { - return std::make_unique(name(), humanName(), helpText(), m_DefaultValue, m_AllowedTypes, m_RequiredComponentShapes); + return std::make_unique(name(), humanName(), helpText(), m_DefaultValue, m_AllowedTypes, m_RequiredComponentShapes, m_Location); } std::any ArraySelectionParameter::defaultValue() const @@ -202,15 +205,34 @@ Result<> ArraySelectionParameter::validatePath(const DataStructure& dataStructur if(m_Location != DataLocation::Any) { - IDataStore::StoreType storeType = dataArray->getStoreType(); - - if(storeType == IDataStore::StoreType::Empty) + const IDataStore::StoreType storeType = dataArray->getStoreType(); + const bool matches = + (m_Location == DataLocation::InMemory && storeType == IDataStore::StoreType::InMemory) || (m_Location == DataLocation::OutOfCore && storeType == IDataStore::StoreType::OutOfCore); + if(storeType != IDataStore::StoreType::Empty && !matches) { - return {}; - } + const char* actualLocationNamePtr = "unsupported"; + if(storeType == IDataStore::StoreType::InMemory) + { + actualLocationNamePtr = "in-memory"; + } + else if(storeType == IDataStore::StoreType::OutOfCore) + { + actualLocationNamePtr = "out-of-core"; + } - return MakeErrorResult(FilterParameter::Constants::k_Validate_DataLocation_Error, - fmt::format("{}DataArray at path '{}' was stored at '{}', but only {} are allowed", prefix, value.toString(), fmt::underlying(storeType), fmt::underlying(m_Location))); + const char* requestedLocationNamePtr = "unsupported"; + if(m_Location == DataLocation::InMemory) + { + requestedLocationNamePtr = "in-memory"; + } + else if(m_Location == DataLocation::OutOfCore) + { + requestedLocationNamePtr = "out-of-core"; + } + + return MakeErrorResult(FilterParameter::Constants::k_Validate_DataLocation_Error, + fmt::format("{}DataArray at path '{}' has '{}' storage; required storage is '{}'.", prefix, value.toString(), actualLocationNamePtr, requestedLocationNamePtr)); + } } } return {}; diff --git a/src/simplnx/Pipeline/AbstractPipelineNode.cpp b/src/simplnx/Pipeline/AbstractPipelineNode.cpp index d65e29122b..820b780ab4 100644 --- a/src/simplnx/Pipeline/AbstractPipelineNode.cpp +++ b/src/simplnx/Pipeline/AbstractPipelineNode.cpp @@ -1,12 +1,19 @@ #include "AbstractPipelineNode.hpp" +#include "simplnx/Common/Result.hpp" +#include +#include #include "simplnx/Core/Application.hpp" #include "simplnx/Core/Preferences.hpp" #include "simplnx/Pipeline/Messaging/NodeStatusMessage.hpp" #include "simplnx/Pipeline/Pipeline.hpp" +#include #include +#include +#include + #include using namespace nx::core; @@ -65,7 +72,8 @@ const DataStructure& AbstractPipelineNode::getDataStructure() const void AbstractPipelineNode::setDataStructure(const DataStructure& dataStructure) { - m_DataStructure = dataStructure; + DataStructure candidate(dataStructure); + m_DataStructure = std::move(candidate); } void AbstractPipelineNode::checkDataStructureSize(DataStructure& dataStructure) @@ -119,6 +127,54 @@ void AbstractPipelineNode::endExecution(DataStructure& dataStructure) setDataStructure(dataStructure); } +bool AbstractPipelineNode::hasCompletionErrors() const +{ + return m_HasCompletionErrors; +} + +void AbstractPipelineNode::setHasCompletionErrors(bool value) +{ + m_HasCompletionErrors = value; +} + +Result<> AbstractPipelineNode::endExecutionChecked(DataStructure& dataStructure) +{ + auto result = dataStructure.flushChecked(); + if(result.invalid() && result.errors().empty()) + { + result.errors().push_back({.code = -6071, .message = fmt::format("Node '{}' checked flush failed without an error diagnostic.", getName())}); + } + ErrorCollection snapshotErrors; + try + { + setDataStructure(dataStructure); + } catch(const std::bad_alloc&) + { + snapshotErrors.push_back({.code = -272, .message = fmt::format("Node '{}' completion snapshot failed because memory allocation failed.", getName())}); + } catch(const std::exception& exception) + { + snapshotErrors.push_back({.code = -6071, .message = fmt::format("Node '{}' completion snapshot failed: {}", getName(), exception.what())}); + } catch(...) + { + snapshotErrors.push_back({.code = -6071, .message = fmt::format("Node '{}' completion snapshot failed with an unknown exception.", getName())}); + } + if(!snapshotErrors.empty()) + { + if(result.valid()) + { + result.m_Expected = nonstd::make_unexpected(std::move(snapshotErrors)); + } + else + { + for(auto& error : snapshotErrors) + { + result.errors().push_back(std::move(error)); + } + } + } + return result; +} + void AbstractPipelineNode::notify(const std::shared_ptr& msg) { m_Signal(this, msg); diff --git a/src/simplnx/Pipeline/AbstractPipelineNode.hpp b/src/simplnx/Pipeline/AbstractPipelineNode.hpp index f75f8221a2..b0a2c19872 100644 --- a/src/simplnx/Pipeline/AbstractPipelineNode.hpp +++ b/src/simplnx/Pipeline/AbstractPipelineNode.hpp @@ -68,6 +68,12 @@ class SIMPLNX_EXPORT AbstractPipelineNode using FilterFaultDetailSignalType = nod::signal; const FilterFaultDetailSignalType& getFilterFaultDetailSignal() const; FilterFaultDetailSignalType& getFilterFaultDetailSignal(); + /** + * @brief Sends node diagnostics. Index -1 identifies pipeline completion without a filter row. + * @param filterIndex Filter row, or -1 for pipeline completion. + * @param warnings Warning details in execution order. + * @param errors Error details in execution order. + */ void sendFilterFaultDetailMessage(int32_t filterIndex, const WarningCollection& warnings, const ErrorCollection& errors); using CancelledSignalType = nod::signal; @@ -157,6 +163,13 @@ class SIMPLNX_EXPORT AbstractPipelineNode */ FaultState getFaultState() const; + /** + * @brief Reports completion failure from the last actual execution. + * @return True if checked persistence or snapshot publication failed. + * @note Preflight and rejected execution requests do not reset this outcome. + */ + bool hasCompletionErrors() const; + /** * @brief Returns true if the node has errors. Otherwise, this method returns * false. @@ -283,9 +296,9 @@ class SIMPLNX_EXPORT AbstractPipelineNode AbstractPipelineNode(Pipeline* parent = nullptr); /** - * @brief Updates the stored DataStructure. This should only be called from - * within the execute(DataStructure&) method. - * @param dataStructure + * @brief Publishes a shallow snapshot after the complete copy succeeds. + * @param dataStructure Source whose stores remain shared with the snapshot. + * @note Copy failure preserves the previous snapshot. */ void setDataStructure(const DataStructure& dataStructure); @@ -304,12 +317,27 @@ class SIMPLNX_EXPORT AbstractPipelineNode void setPreflightStructure(const DataStructure& dataStructure, bool success = true); /** - * @brief Called when ending pipeline node execution. - * Sets the DataStructure and clears the Executing flag. - * If there is a parent node, sets the Executed flag. + * @brief Flushes and snapshots data through the legacy completion interface. + * @param dataStructure Completed execution data. + * @note Checked pipeline execution calls endExecutionChecked instead of this method. */ virtual void endExecution(DataStructure& dataStructure); + /** + * @brief Checks persistence and attempts a shallow snapshot even if persistence fails. + * @param dataStructure Completed execution data. + * @return Flush diagnostics followed by snapshot diagnostics, if any. + * @note Extensions that customize checked completion must override this method. + * @throws std::bad_alloc If diagnostic allocation fails. + */ + [[nodiscard]] virtual Result<> endExecutionChecked(DataStructure& dataStructure); + + /** + * @brief Records checked completion failure independently of execute-stage cancellation. + * @param value Completion outcome for the current actual execution. + */ + void setHasCompletionErrors(bool value); + /** * @brief Returns a Pipeline containing the parent pipeline up to the current * node. This will expand DREAM3D files as their own Pipeline. @@ -339,6 +367,7 @@ class SIMPLNX_EXPORT AbstractPipelineNode DataStructure m_DataStructure; DataStructure m_PreflightStructure; bool m_IsPreflighted = false; + bool m_HasCompletionErrors = false; SignalType m_Signal; FaultState m_FaultState = FaultState::None; bool m_IsDisabled = false; diff --git a/src/simplnx/Pipeline/Pipeline.cpp b/src/simplnx/Pipeline/Pipeline.cpp index c728cfe3e9..f779190dae 100644 --- a/src/simplnx/Pipeline/Pipeline.cpp +++ b/src/simplnx/Pipeline/Pipeline.cpp @@ -1,4 +1,5 @@ #include "Pipeline.hpp" +#include "simplnx/Common/Result.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/Filter/FilterHandle.hpp" @@ -11,8 +12,12 @@ #include "simplnx/Pipeline/PipelineFilter.hpp" #include "simplnx/Pipeline/PlaceholderFilter.hpp" +#include #include +#include +#include + #include #include #include @@ -316,6 +321,7 @@ bool Pipeline::executeFrom(index_type index, DataStructure& dataStructure, const { return false; } + setHasCompletionErrors(false); bool returnValue = true; // Send notification that the pipeline is executing sendPipelineRunStateMessage(RunState::Executing); @@ -345,15 +351,21 @@ bool Pipeline::executeFrom(index_type index, DataStructure& dataStructure, const } bool success = filter->execute(dataStructure, shouldCancel); - // Check if the filter was cancelled, and send out signal if it was. + setHasWarnings(hasWarnings() || filter->hasWarnings()); + if(filter->hasCompletionErrors()) + { + setHasCompletionErrors(true); + setHasErrors(); + returnValue = false; + } + // Execute-stage cancellation keeps its existing outcome; checked completion failure must survive it. if(shouldCancel) { sendCancelledMessage(); break; } - setHasWarnings(filter->hasWarnings()); - if(!success) + if(!success || filter->hasCompletionErrors()) { setHasErrors(); returnValue = false; @@ -362,7 +374,27 @@ bool Pipeline::executeFrom(index_type index, DataStructure& dataStructure, const } // checkDataStructureSize(dataStructure); - setDataStructure(dataStructure); + ErrorCollection snapshotErrors; + try + { + setDataStructure(dataStructure); + } catch(const std::bad_alloc&) + { + snapshotErrors.push_back({.code = -272, .message = fmt::format("Pipeline '{}' final completion snapshot failed because memory allocation failed.", getName())}); + } catch(const std::exception& exception) + { + snapshotErrors.push_back({.code = -6071, .message = fmt::format("Pipeline '{}' final completion snapshot failed: {}", getName(), exception.what())}); + } catch(...) + { + snapshotErrors.push_back({.code = -6071, .message = fmt::format("Pipeline '{}' final completion snapshot failed with an unknown exception.", getName())}); + } + if(!snapshotErrors.empty()) + { + setHasCompletionErrors(true); + setHasErrors(); + returnValue = false; + sendFilterFaultDetailMessage(-1, {}, snapshotErrors); + } sendPipelineFaultMessage(m_FaultState); sendPipelineRunStateMessage(RunState::Idle); diff --git a/src/simplnx/Pipeline/PipelineFilter.cpp b/src/simplnx/Pipeline/PipelineFilter.cpp index de5dd15d28..ca5d283820 100644 --- a/src/simplnx/Pipeline/PipelineFilter.cpp +++ b/src/simplnx/Pipeline/PipelineFilter.cpp @@ -1,4 +1,6 @@ #include "PipelineFilter.hpp" +#include +#include #include "simplnx/Core/Application.hpp" #include "simplnx/Filter/FilterList.hpp" @@ -254,6 +256,7 @@ bool PipelineFilter::preflight(DataStructure& dataStructure, RenamedPaths& renam // ----------------------------------------------------------------------------- bool PipelineFilter::execute(DataStructure& dataStructure, const std::atomic_bool& shouldCancel) { + setHasCompletionErrors(false); this->sendFilterRunStateMessage(m_Index, nx::core::RunState::Executing); this->sendFilterUpdateMessage(m_Index, "Begin"); @@ -284,9 +287,26 @@ bool PipelineFilter::execute(DataStructure& dataStructure, const std::atomic_boo m_Errors.push_back(Error{-11, "This filter is just a placeholder! The original filter could not be found. See the filter comments for more details."}); } + auto completionResult = endExecutionChecked(dataStructure); + setHasCompletionErrors(completionResult.invalid()); + for(auto& warning : completionResult.warnings()) + { + m_Warnings.push_back(std::move(warning)); + } + if(completionResult.invalid()) + { + if(completionResult.errors().empty()) + { + completionResult.errors().push_back({.code = -6071, .message = fmt::format("Node '{}' checked completion failed without an error diagnostic.", getName())}); + } + for(auto& error : completionResult.errors()) + { + m_Errors.push_back(std::move(error)); + } + } + const bool succeeded = result.result.valid() && completionResult.valid() && m_Filter != nullptr; setHasWarnings(!m_Warnings.empty()); - setHasErrors(!m_Errors.empty()); - endExecution(dataStructure); + setHasErrors(!succeeded); if(!m_Warnings.empty() || !m_Errors.empty()) { @@ -296,7 +316,7 @@ bool PipelineFilter::execute(DataStructure& dataStructure, const std::atomic_boo this->sendFilterRunStateMessage(m_Index, nx::core::RunState::Idle); this->sendFilterUpdateMessage(m_Index, "End"); - return result.result.valid() && m_Filter != nullptr; + return succeeded; } std::vector PipelineFilter::getCreatedPaths() const diff --git a/src/simplnx/Utilities/AlgorithmDispatch.hpp b/src/simplnx/Utilities/AlgorithmDispatch.hpp index 55ef70f72c..10522a6bc0 100644 --- a/src/simplnx/Utilities/AlgorithmDispatch.hpp +++ b/src/simplnx/Utilities/AlgorithmDispatch.hpp @@ -196,6 +196,20 @@ class ForceOocAlgorithmGuard */ SIMPLNX_EXPORT bool& ForceInCoreAlgorithm(); +/** + * @brief Selects the algorithm path from storage residency and test flags. + * @param usesOutOfCoreStore True when a relevant target uses out-of-core storage. + * @return True when the out-of-core algorithm is selected. + * + * ForceInCoreAlgorithm() overrides storage residency and ForceOocAlgorithm(). + * Callers retain storage capability checks and record the path that runs. + * @warning Set the process-wide test flags before parallel work starts. + */ +inline bool ShouldUseOutOfCoreAlgorithm(bool usesOutOfCoreStore) +{ + return !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); +} + /** * @class ForceInCoreAlgorithmGuard * @brief Restores the force-in-core test flag at scope exit. @@ -319,7 +333,7 @@ template Result<> DispatchAlgorithm(std::initializer_list arrays, ArgsT&&... args) { const bool usesOutOfCoreStore = AnyOutOfCore(arrays); - const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + const bool useOutOfCoreAlgorithm = ShouldUseOutOfCoreAlgorithm(usesOutOfCoreStore); RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); if(useOutOfCoreAlgorithm) @@ -345,7 +359,7 @@ template Result<> DispatchAlgorithm(const AlgorithmArrayTargets& targets, ArgsT&&... args) { const bool usesOutOfCoreStore = AnyOutOfCore(targets); - const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + const bool useOutOfCoreAlgorithm = ShouldUseOutOfCoreAlgorithm(usesOutOfCoreStore); RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); if(useOutOfCoreAlgorithm) diff --git a/src/simplnx/Utilities/ArrayCreationUtilities.hpp b/src/simplnx/Utilities/ArrayCreationUtilities.hpp index 636e252646..7bdad27052 100644 --- a/src/simplnx/Utilities/ArrayCreationUtilities.hpp +++ b/src/simplnx/Utilities/ArrayCreationUtilities.hpp @@ -248,14 +248,7 @@ Result<> CreateArray(DataStructure& dataStructure, const ShapeType& tupleShape, if(mode == IDataAction::Mode::Execute) { store->fill(conversionResult.value()); - { - // Only resident DataStore records an initialization value for later resize operations. - std::weak_ptr> weakDataStorePtr = std::dynamic_pointer_cast>(store); - if(auto dataStorePtr = weakDataStorePtr.lock(); dataStorePtr != nullptr) - { - dataStorePtr->setInitValue(conversionResult.value()); - } - } + store->setInitValue(conversionResult.value()); } } diff --git a/src/simplnx/Utilities/CacheMemoryBudgetManager.cpp b/src/simplnx/Utilities/CacheMemoryBudgetManager.cpp index e7642f5f29..8b3dca02bd 100644 --- a/src/simplnx/Utilities/CacheMemoryBudgetManager.cpp +++ b/src/simplnx/Utilities/CacheMemoryBudgetManager.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include "simplnx/Utilities/MemoryUtilities.hpp" @@ -134,7 +135,7 @@ std::pair lock(m_Mutex); + const std::lock_guard lock(m_Mutex); std::vector evicted = makeRoom(sizeBytes); @@ -159,7 +160,7 @@ std::pair CacheMemoryBudgetManager::reservePinned(const std::string& subsystem, const std::string& key, uint64 sizeBytes) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); if(m_PinnedBytes > m_BudgetBytes || sizeBytes > m_BudgetBytes - m_PinnedBytes) { return std::nullopt; @@ -182,7 +183,7 @@ std::optional CacheMemoryBudgetManag bool CacheMemoryBudgetManager::pin(AllocationHandle handle) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); auto iter = m_Entries.find(handle); if(iter == m_Entries.end()) { @@ -200,7 +201,7 @@ bool CacheMemoryBudgetManager::pin(AllocationHandle handle) CacheMemoryBudgetManager::PinResult CacheMemoryBudgetManager::pinWithinBudget(AllocationHandle handle) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); auto iter = m_Entries.find(handle); if(iter == m_Entries.end()) { @@ -222,7 +223,7 @@ CacheMemoryBudgetManager::PinResult CacheMemoryBudgetManager::pinWithinBudget(Al bool CacheMemoryBudgetManager::unpin(AllocationHandle handle) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); auto iter = m_Entries.find(handle); if(iter == m_Entries.end()) { @@ -245,7 +246,7 @@ bool CacheMemoryBudgetManager::unpin(AllocationHandle handle) void CacheMemoryBudgetManager::touch(AllocationHandle handle) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); auto it = m_Entries.find(handle); if(it != m_Entries.end()) { @@ -255,13 +256,13 @@ void CacheMemoryBudgetManager::touch(AllocationHandle handle) void CacheMemoryBudgetManager::registerSubsystem(const std::string& subsystem, SubsystemEvictionHandler handler) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); m_SubsystemHandlers[subsystem] = std::move(handler); } void CacheMemoryBudgetManager::release(AllocationHandle handle) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); auto it = m_Entries.find(handle); if(it != m_Entries.end()) { @@ -276,7 +277,7 @@ void CacheMemoryBudgetManager::release(AllocationHandle handle) CacheMemoryBudgetManager::WorkingMemoryReservation CacheMemoryBudgetManager::reserveWorkingMemory(uint64 requestedBytes) { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); const uint64 maximumWorkingBytes = m_BudgetBytes / 4; const uint64 remainingWorkingBytes = maximumWorkingBytes > m_ReservedWorkingMemoryBytes ? maximumWorkingBytes - m_ReservedWorkingMemoryBytes : 0; const uint64 grantedBytes = std::min(requestedBytes, remainingWorkingBytes); @@ -290,6 +291,23 @@ CacheMemoryBudgetManager::WorkingMemoryReservation CacheMemoryBudgetManager::res return WorkingMemoryReservation(WorkingMemoryReservationKey{}, this, grantedBytes); } +CacheMemoryBudgetManager::WorkingMemoryReservation CacheMemoryBudgetManager::tryReserveWorkingMemoryWithoutEviction(uint64 requestedBytes) +{ + const std::lock_guard lock(m_Mutex); + const uint64 maximumWorkingBytes = m_BudgetBytes / 4; + if(requestedBytes == 0 || m_ReservedWorkingMemoryBytes > maximumWorkingBytes || requestedBytes > maximumWorkingBytes - m_ReservedWorkingMemoryBytes) + { + return {}; + } + if(m_UsedBytes > m_BudgetBytes || m_ReservedWorkingMemoryBytes > m_BudgetBytes - m_UsedBytes || requestedBytes > m_BudgetBytes - m_UsedBytes - m_ReservedWorkingMemoryBytes) + { + return {}; + } + + m_ReservedWorkingMemoryBytes += requestedBytes; + return WorkingMemoryReservation(WorkingMemoryReservationKey{}, this, requestedBytes); +} + bool CacheMemoryBudgetManager::setBudgetBytes(uint64 bytes) { // Clamp only the upper bound. Calculate the machine limit before acquiring m_Mutex. @@ -301,50 +319,50 @@ bool CacheMemoryBudgetManager::setBudgetBytes(uint64 bytes) clamped = true; } - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); m_BudgetBytes = bytes; return clamped; } uint64 CacheMemoryBudgetManager::budgetBytes() const { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); return m_BudgetBytes; } uint64 CacheMemoryBudgetManager::usedBytes() const { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); return m_UsedBytes; } uint64 CacheMemoryBudgetManager::pinnedBytes() const { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); return m_PinnedBytes; } uint64 CacheMemoryBudgetManager::maximumWorkingMemoryBytes() const { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); return m_BudgetBytes / 4; } uint64 CacheMemoryBudgetManager::reservedWorkingMemoryBytes() const { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); return m_ReservedWorkingMemoryBytes; } uint64 CacheMemoryBudgetManager::effectiveCacheBudgetBytes() const { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); return m_BudgetBytes > m_ReservedWorkingMemoryBytes ? m_BudgetBytes - m_ReservedWorkingMemoryBytes : 0; } void CacheMemoryBudgetManager::clear() { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); m_Entries.clear(); m_UsedBytes = 0; m_PinnedBytes = 0; @@ -410,7 +428,7 @@ std::vector CacheMemoryBudgetManager void CacheMemoryBudgetManager::releaseWorkingMemory(uint64 sizeBytes) noexcept { - std::lock_guard lock(m_Mutex); + const std::lock_guard lock(m_Mutex); m_ReservedWorkingMemoryBytes = sizeBytes <= m_ReservedWorkingMemoryBytes ? m_ReservedWorkingMemoryBytes - sizeBytes : 0; } diff --git a/src/simplnx/Utilities/CacheMemoryBudgetManager.hpp b/src/simplnx/Utilities/CacheMemoryBudgetManager.hpp index bc03f6b59c..243a0f6de0 100644 --- a/src/simplnx/Utilities/CacheMemoryBudgetManager.hpp +++ b/src/simplnx/Utilities/CacheMemoryBudgetManager.hpp @@ -35,8 +35,8 @@ namespace nx::core * buffers may reserve cache headroom through WorkingMemoryReservation without * being reported as cache usage. * - * Thread-safe: allocate/touch/release/reserveWorkingMemory can be called from - * any thread. + * Thread-safe: Cache registrations and working-memory reservations can be made + * from any thread. * * Eviction callbacks and delegated handlers run under the manager mutex. They * can only update an atomic or a leaf mark queue. They must not re-enter this manager. @@ -253,6 +253,20 @@ class SIMPLNX_EXPORT CacheMemoryBudgetManager */ [[nodiscard]] WorkingMemoryReservation reserveWorkingMemory(uint64 requestedBytes); + /** + * @brief Reserves all requested working bytes when registered cache headroom is available. + * @param requestedBytes Number of bytes to reserve. + * @return Full token, or empty token on + * failure. + * + * The request must fit the one-quarter working limit and remaining registered + * cache headroom. This method does not request eviction. + * + * Direct eviction can defer + * physical release beyond accounting removal. + */ + [[nodiscard]] WorkingMemoryReservation tryReserveWorkingMemoryWithoutEviction(uint64 requestedBytes); + /** * @brief Sets the cache memory budget in bytes, clamped to maxBudgetBytes(). * diff --git a/src/simplnx/Utilities/GeometryUtilities.cpp b/src/simplnx/Utilities/GeometryUtilities.cpp index b375f245e2..4ab6cead6e 100644 --- a/src/simplnx/Utilities/GeometryUtilities.cpp +++ b/src/simplnx/Utilities/GeometryUtilities.cpp @@ -1,7 +1,10 @@ #include "GeometryUtilities.hpp" +#include "simplnx/DataStructure/IDataStore.hpp" #include "simplnx/Common/Array.hpp" #include "simplnx/Common/Result.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" +#include "simplnx/Utilities/Meshing/TriangleUtilities.hpp" #include #include @@ -248,6 +251,21 @@ class CalculateNormalsImpl Result<> GeometryUtilities::ComputeTriangleNormals(const nx::core::TriangleGeom* triangleGeom, Float64AbstractDataStore& normals, const std::atomic_bool& shouldCancel) { + const bool usesOutOfCoreStore = AnyOutOfCore({triangleGeom->getFaces(), triangleGeom->getVertices()}) || normals.getStoreType() == IDataStore::StoreType::OutOfCore; + const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + // Preserve this utility's float arithmetic, which differs from the Eigen double path. + return MeshingUtilities::detail::CalculateNormalsInBlocks(triangleGeom->getFaces()->getDataStoreRef(), triangleGeom->getVertices()->getDataStoreRef(), normals, shouldCancel, + [](const float32* a, const float32* b, const float32* c) -> std::array { + const std::array vertCoords = {Point3Df{a[0], a[1], a[2]}, Point3Df{b[0], b[1], b[2]}, Point3Df{c[0], c[1], c[2]}}; + auto normal = (vertCoords[1] - vertCoords[0]).cross(vertCoords[2] - vertCoords[0]); + normal = normal / normal.magnitude(); + return {static_cast(normal[0]), static_cast(normal[1]), static_cast(normal[2])}; + }); + } + // Parallel algorithm to find duplicate nodes ParallelDataAlgorithm dataAlg; dataAlg.setRange(0ULL, static_cast(triangleGeom->getNumberOfFaces())); diff --git a/src/simplnx/Utilities/IParallelAlgorithm.cpp b/src/simplnx/Utilities/IParallelAlgorithm.cpp index c6e0ccd7ff..571121396c 100644 --- a/src/simplnx/Utilities/IParallelAlgorithm.cpp +++ b/src/simplnx/Utilities/IParallelAlgorithm.cpp @@ -81,12 +81,12 @@ void IParallelAlgorithm::setParallelizationEnabled(bool doParallel) // ----------------------------------------------------------------------------- void IParallelAlgorithm::requireArraysInMemory(const AlgorithmArrays& arrays) { - setParallelizationEnabled(CheckArraysInMemory(arrays)); + setParallelizationEnabled(getParallelizationEnabled() && CheckArraysInMemory(arrays)); } // ----------------------------------------------------------------------------- void IParallelAlgorithm::requireStoresInMemory(const AlgorithmStores& stores) { - setParallelizationEnabled(::CheckStoresInMemory(stores)); + setParallelizationEnabled(getParallelizationEnabled() && ::CheckStoresInMemory(stores)); } } // namespace nx::core diff --git a/src/simplnx/Utilities/IParallelAlgorithm.hpp b/src/simplnx/Utilities/IParallelAlgorithm.hpp index 1ec99a1dde..4608d9ed24 100644 --- a/src/simplnx/Utilities/IParallelAlgorithm.hpp +++ b/src/simplnx/Utilities/IParallelAlgorithm.hpp @@ -27,13 +27,31 @@ class SIMPLNX_EXPORT IParallelAlgorithm [[nodiscard]] bool getParallelizationEnabled() const; /** - * @brief Sets whether parallelization is enabled. - * @param doParallel + * @brief Sets the parallelization state explicitly. + * @param doParallel Requested parallelization state. + * + * When doParallel is true, this call resets restrictions from earlier + * require calls in multicore builds. + * Builds without SIMPLNX_ENABLE_MULTICORE keep parallelization disabled. */ void setParallelizationEnabled(bool doParallel); + /** + * @brief Requires array stores to be in memory for parallel work. + * @param arrays Arrays whose stores must be in memory. + * + * Each call preserves a disabled state. Call + * setParallelizationEnabled(true) to reset the state in a multicore build. + */ void requireArraysInMemory(const AlgorithmArrays& arrays); + /** + * @brief Requires stores to be in memory for parallel work. + * @param arrays Stores that must be in memory. + * + * Each call preserves a disabled state. Call + * setParallelizationEnabled(true) to reset the state in a multicore build. + */ void requireStoresInMemory(const AlgorithmStores& arrays); protected: diff --git a/src/simplnx/Utilities/ImageProcessing/ZeroCrossingEngine.hpp b/src/simplnx/Utilities/ImageProcessing/ZeroCrossingEngine.hpp index 15f3296923..b70de8f834 100644 --- a/src/simplnx/Utilities/ImageProcessing/ZeroCrossingEngine.hpp +++ b/src/simplnx/Utilities/ImageProcessing/ZeroCrossingEngine.hpp @@ -371,7 +371,8 @@ Result<> ApplyZeroCrossing(const AbstractDataStore& inStore, AbstractDataStor const bool usesOutOfCoreStore = inStore.getStoreType() == IDataStore::StoreType::OutOfCore || outStore.getStoreType() == IDataStore::StoreType::OutOfCore; const auto* inMemoryInput = dynamic_cast*>(&inStore); auto* inMemoryOutput = dynamic_cast*>(&outStore); - const bool direct = !usesOutOfCoreStore && inMemoryInput != nullptr && inMemoryOutput != nullptr && !ForceOocAlgorithm(); + // Direct spans require concrete resident stores even when a test requests the in-core route. + const bool direct = !usesOutOfCoreStore && inMemoryInput != nullptr && inMemoryOutput != nullptr && !ShouldUseOutOfCoreAlgorithm(usesOutOfCoreStore); RecordAlgorithmPathExecution(direct ? AlgorithmPath::InCore : AlgorithmPath::OutOfCore, usesOutOfCoreStore); if(shouldCancel) { diff --git a/src/simplnx/Utilities/Meshing/TriangleUtilities.cpp b/src/simplnx/Utilities/Meshing/TriangleUtilities.cpp index f7cd55a5b0..1b1af65dca 100644 --- a/src/simplnx/Utilities/Meshing/TriangleUtilities.cpp +++ b/src/simplnx/Utilities/Meshing/TriangleUtilities.cpp @@ -1,14 +1,27 @@ #include "TriangleUtilities.hpp" - +#include "simplnx/DataStructure/Geometry/INodeGeometry2D.hpp" +#include "simplnx/DataStructure/IDataStore.hpp" +#include +#include + +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" #include "simplnx/DataStructure/IO/Generic/IExternalSort.hpp" #include "simplnx/DataStructure/IO/Generic/ITemporaryRecordStore.hpp" +#include "simplnx/Utilities/AlgorithmDispatch.hpp" #include "simplnx/Utilities/BoundedRecordPageCache.hpp" +#include "simplnx/Utilities/CacheMemoryBudgetManager.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" +#include "simplnx/Utilities/Meshing/VertexUtilities.hpp" +#include "simplnx/Utilities/ParallelDataAlgorithm.hpp" #include "simplnx/Utilities/ThrottledMessageHandler.hpp" #include #include +#include #include #include @@ -18,6 +31,7 @@ #include #include #include +#include #include #include @@ -26,6 +40,10 @@ using namespace nx::core; namespace { using EdgeListT = std::set>; +using WindingFaceType = std::array; + +static_assert(std::is_trivially_copyable_v); +static_assert(sizeof(WindingFaceType) == 3 * sizeof(IGeometry::MeshIndexType)); inline constexpr uint64 k_WindingBatchRecords = 4096; inline constexpr usize k_WindingCachePages = 8; @@ -398,7 +416,7 @@ class WindingSortReader /** * @class WindingDataStoreCache - * @brief Provides bounded LRU tuple access to one DataStore. + * @brief Provides bounded read-only LRU tuple access to one DataStore. * @tparam T Specifies the scalar type. */ template @@ -418,20 +436,6 @@ class WindingDataStoreCache { } - /** - * @brief Creates a mutable tuple cache. - * @param store Provides and receives tuples. - * @param tuplesPerPage Specifies page tuple count. - * @param maximumPages Limits resident pages. - */ - WindingDataStoreCache(AbstractDataStore& store, uint64 tuplesPerPage, usize maximumPages) - : m_Store(store) - , m_MutableStore(&store) - , m_TuplesPerPage(tuplesPerPage) - , m_MaximumPages(maximumPages) - { - } - /** * @brief Reads one tuple through the cache. * @param tupleIndex Specifies the tuple. @@ -455,61 +459,15 @@ class WindingDataStoreCache return {}; } - /** - * @brief Updates one cached tuple and marks its page dirty. - * @param tupleIndex Specifies the tuple. - * @param tuple Provides all tuple components. - * @return Mutability, shape, range, allocation, or source-read error, or success. - */ - Result<> writeTuple(uint64 tupleIndex, nonstd::span tuple) - { - if(m_MutableStore == nullptr) - { - return MakeErrorResult(-65787, "Triangle winding attempted to modify a read-only DataStore cache."); - } - if(tuple.size() != m_Store.getNumberOfComponents()) - { - return MakeErrorResult(-65786, "Triangle winding DataStore cache received a tuple with the wrong component count."); - } - auto pageResult = loadPage(tupleIndex); - if(pageResult.invalid()) - { - return ConvertResult(std::move(pageResult)); - } - Page& page = pageResult.value().get(); - const usize localTuple = static_cast(tupleIndex - page.FirstTuple); - std::copy(tuple.begin(), tuple.end(), page.Values.data() + localTuple * tuple.size()); - page.Dirty = true; - return {}; - } - - /** - * @brief Flushes all dirty pages. - * @return First destination-write error, or success. - */ - Result<> flush() - { - for(Page& page : m_Pages) - { - auto result = flushPage(page); - if(result.invalid()) - { - return result; - } - } - return {}; - } - private: /** * @struct Page - * @brief Stores one tuple page and its dirty state. + * @brief Stores one read-only tuple page. */ struct Page { uint64 FirstTuple = 0; uint64 TupleCount = 0; - bool Dirty = false; std::vector Values; }; @@ -535,11 +493,6 @@ class WindingDataStoreCache } if(m_Pages.size() == m_MaximumPages) { - auto flushResult = flushPage(m_Pages.back()); - if(flushResult.invalid()) - { - return ConvertInvalidResult>(std::move(flushResult)); - } m_Pages.pop_back(); } Page page; @@ -568,36 +521,57 @@ class WindingDataStoreCache return {std::ref(m_Pages.front())}; } - /** - * @brief Writes one dirty page to the mutable store. - * @param page Provides cached values and destination range. - * @return Mutability or destination-write error, or success. - */ - Result<> flushPage(Page& page) + const AbstractDataStore& m_Store; + uint64 m_TuplesPerPage = 0; + usize m_MaximumPages = 0; + std::list m_Pages; +}; + +/** + * @brief Copies repaired scratch connectivity back in bounded sequential batches. + * @param triangles Receives the repaired connectivity. + * @param faceStore Provides the scratch connectivity. + * @param faceCache Provides pending scratch writes. + * @param buffer Reuses the initial connectivity scan buffer. + * @param facesModified Skips destination writes when no triangle was flipped. + * @return Scratch flush/read or destination-write error, or success. + * @pre buffer contains at least k_WindingBatchRecords * 3 elements. + */ +Result<> WriteWindingFaces(INodeGeometry2D::SharedFaceList::store_type& triangles, ITemporaryRecordStore& faceStore, BoundedRecordPageCache& faceCache, + nonstd::span buffer, bool facesModified) +{ + if(!facesModified) + { + return {}; + } + // Preserve completed flips when traversal is cancelled. + const std::atomic_bool neverCancel = false; + auto flushResult = faceCache.flush(neverCancel); + if(flushResult.invalid()) + { + return flushResult; + } + for(uint64 offset = 0; offset < faceStore.recordCount(); offset += k_WindingBatchRecords) { - if(!page.Dirty) + const uint64 count = std::min(k_WindingBatchRecords, faceStore.recordCount() - offset); + auto bytes = nonstd::span(reinterpret_cast(buffer.data()), static_cast(count * sizeof(WindingFaceType))); + auto readResult = faceStore.read(offset, count, bytes, neverCancel); + if(readResult.invalid()) { - return {}; + return ConvertResult(std::move(readResult)); } - if(m_MutableStore == nullptr) + if(readResult.value() != count) { - return MakeErrorResult(-65787, "Triangle winding attempted to flush a read-only DataStore cache."); + return MakeErrorResult(-65795, fmt::format("Triangle winding scratch connectivity read at triangle {} returned {} records; expected {}.", offset, readResult.value(), count)); } - const usize componentCount = m_Store.getNumberOfComponents(); - auto result = m_MutableStore->copyFromBuffer(static_cast(page.FirstTuple) * componentCount, nonstd::span(page.Values.data(), page.Values.size())); - if(result.valid()) + auto writeResult = triangles.copyFromBuffer(static_cast(offset * 3), nonstd::span(buffer.data(), static_cast(count * 3))); + if(writeResult.invalid()) { - page.Dirty = false; + return writeResult; } - return result; } - - const AbstractDataStore& m_Store; - AbstractDataStore* m_MutableStore = nullptr; - uint64 m_TuplesPerPage = 0; - usize m_MaximumPages = 0; - std::list m_Pages; -}; + return {}; +} /** * @brief Finalizes one external winding sort. @@ -1037,6 +1011,16 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared WindingSortAppender occurrenceAppender(*occurrences, shouldCancel); WindingSortAppender seedAppender(*seeds, shouldCancel); + // Stage connectivity during the existing scan. Dirty page evictions then target + // uncompressed scratch records, not HDF5 chunks that the next read must compress + // and inflate again. Copy back only when traversal completes or is cancelled. + auto faceStoreResult = CreateWindingRecordStore(triangleCount); + if(faceStoreResult.invalid()) + { + return ConvertResult(std::move(faceStoreResult)); + } + const std::unique_ptr faceStore = std::move(faceStoreResult.value()); + std::vector faceBuffer; std::vector idBuffer; try @@ -1064,6 +1048,12 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared { return faceReadResult; } + auto faceStageResult = + faceStore->write(offset, count, nonstd::span(reinterpret_cast(faceBuffer.data()), static_cast(count * sizeof(WindingFaceType))), shouldCancel); + if(faceStageResult.invalid()) + { + return faceStageResult; + } auto idReadResult = idsStore.copyIntoBuffer(static_cast(offset * componentCount), nonstd::span(idBuffer.data(), static_cast(count * componentCount))); if(idReadResult.invalid()) { @@ -1331,7 +1321,10 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared neighborOffset = groupEnd; } - WindingDataStoreCache faceCache(triangles, k_WindingBatchRecords, k_WindingCachePages); + BoundedRecordPageCache faceCache(*faceStore, k_WindingBatchRecords, k_WindingCachePages); + // Keep face access non-cancelling, as before; traversal owns cancellation checks. + const std::atomic_bool neverCancel = false; + bool facesModified = false; WindingDataStoreCache idCache(idsStore, k_WindingBatchRecords, k_WindingCachePages); WindingSortReader seedReader(*seeds, shouldCancel); @@ -1346,7 +1339,13 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared return {labels[0] == feature || (componentCount == 2 && labels[1] == feature)}; }; const auto readFace = [&](uint64 triangle, std::array& face) -> Result<> { - return faceCache.readTuple(triangle, nonstd::span(face.data(), face.size())); + auto readResult = faceCache.read(triangle, neverCancel); + if(readResult.invalid()) + { + return ConvertResult(std::move(readResult)); + } + face = readResult.value(); + return {}; }; uint64 unrepairedCount = 0; @@ -1357,7 +1356,7 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared { if(shouldCancel) { - return faceCache.flush(); + return WriteWindingFaces(triangles, *faceStore, faceCache, nonstd::span(faceBuffer.data(), faceBuffer.size()), facesModified); } auto seedRecordResult = seedReader.read(seedOffset); if(seedRecordResult.invalid()) @@ -1452,7 +1451,7 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared { if(shouldCancel) { - return faceCache.flush(); + return WriteWindingFaces(triangles, *faceStore, faceCache, nonstd::span(faceBuffer.data(), faceBuffer.size()), facesModified); } auto queuedResult = queueCache.read(queueHead, shouldCancel); if(queuedResult.invalid()) @@ -1560,11 +1559,12 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared else { std::swap(face[0], face[2]); - auto faceWriteResult = faceCache.writeTuple(triangle, nonstd::span(face.data(), face.size())); + auto faceWriteResult = faceCache.write(triangle, face, neverCancel); if(faceWriteResult.invalid()) { return faceWriteResult; } + facesModified = true; } } } @@ -1572,7 +1572,7 @@ Result<> MeshingUtilities::RepairTriangleWindingExternal(INodeGeometry2D::Shared seedOffset = nextFeatureOffset; } - auto faceFlushResult = faceCache.flush(); + auto faceFlushResult = WriteWindingFaces(triangles, *faceStore, faceCache, nonstd::span(faceBuffer.data(), faceBuffer.size()), facesModified); if(faceFlushResult.invalid()) { return faceFlushResult; @@ -1628,6 +1628,301 @@ void MeshingUtilities::CalculateNormalsImpl::operator()(const nx::core::Range& r generate(range.min(), range.max()); } +namespace +{ +constexpr usize k_NormalBlockTriangles = 262144; +constexpr usize k_NormalFastBytesPerFace = (3 * sizeof(IGeometry::MeshIndexType)) + (3 * sizeof(float64)); +constexpr usize k_NormalGatherBytesPerFace = k_NormalFastBytesPerFace + (3 * sizeof(usize)) + (18 * sizeof(float32)); +static_assert(k_NormalBlockTriangles <= std::numeric_limits::max() / k_NormalGatherBytesPerFace); + +struct TriangleNormalWorkingPlan +{ + bool CacheVertices = false; + usize FaceCapacity = 0; + usize PageVertices = 0; + usize PayloadBytes = 0; +}; + +/** + * @brief Plans coordinate and face buffers within the supplied byte limit. + * @param triangles Supplies the face count for connectivity and normal buffers. + * @param vertices Supplies the vertex count for a coordinate cache or read page. + * @param availableBytes Limits the combined buffer payload in bytes. + * @return Plan with its total payload, or zero face capacity when no face fits. + * @pre The caller checks three-component shapes and addressable coordinate and face sizes. + * @note The caller must reserve working memory and plan again from the actual grant before allocating buffers. + */ +TriangleNormalWorkingPlan planNormalWorkingMemory(const AbstractDataStore& triangles, const Float32AbstractDataStore& vertices, usize availableBytes) +{ + const usize maximumTriangles = std::min(k_NormalBlockTriangles, triangles.getNumberOfTuples()); + const usize vertexCount = vertices.getNumberOfTuples(); + const usize vertexBytes = vertexCount * 3 * sizeof(float32); + const usize gatherTriangles = std::min(maximumTriangles, availableBytes / k_NormalGatherBytesPerFace); + const usize admissionTriangles = std::max(usize{1}, gatherTriangles); + // Retaining coordinates must preserve the gathered route's available face capacity. + const bool cacheVertices = vertexBytes <= availableBytes && admissionTriangles * k_NormalFastBytesPerFace <= availableBytes - vertexBytes; + const usize faceCapacity = cacheVertices ? std::min(maximumTriangles, (availableBytes - vertexBytes) / k_NormalFastBytesPerFace) : gatherTriangles; + if(faceCapacity == 0) + { + return {}; + } + const usize pageVertices = cacheVertices ? 0 : std::min(vertexCount, faceCapacity * 3); + const usize coordinateBytes = cacheVertices ? vertexBytes : pageVertices * 3 * sizeof(float32); + const usize bytesPerFace = k_NormalFastBytesPerFace + (cacheVertices ? 0 : (3 * sizeof(usize)) + (9 * sizeof(float32))); + if(coordinateBytes > availableBytes || faceCapacity > (availableBytes - coordinateBytes) / bytesPerFace) + { + return {}; + } + return {.CacheVertices = cacheVertices, .FaceCapacity = faceCapacity, .PageVertices = pageVertices, .PayloadBytes = coordinateBytes + (faceCapacity * bytesPerFace)}; +} + +/** + * @brief Builds the error for a working grant that cannot fund one face. + * @param code Distinguishes the initial admission from the admission after the first-normal publication. + * @param requestedBytes Bytes requested from the working-memory budget. + * @param grantedBytes Bytes actually granted. + * @param vertexBytes Bytes needed to cache every vertex coordinate. + * @return Error result with the requested, granted, and minimum byte counts. + */ +Result<> MakeNormalAdmissionError(int32 code, uint64 requestedBytes, usize grantedBytes, usize vertexBytes) +{ + return MakeErrorResult(code, fmt::format("Triangle normal calculation requested {} working bytes and received {}. At least {} bytes are needed for one gathered face, or {} bytes for cached " + "vertices and one face. Retry after other operations finish.", + requestedBytes, grantedBytes, k_NormalGatherBytesPerFace, vertexBytes + k_NormalFastBytesPerFace)); +} + +/** + * @brief Calculates the normal of triangle 0 from its inputs and writes it before compute buffers are admitted. + * @param triangles Provides the three vertex indexes of triangle 0. + * @param verts Provides the XYZ coordinates of those vertices. + * @param normals Receives the normal of triangle 0. + * @param calculateNormal Preserves the caller's arithmetic using three resident XYZ pointers. + * @return Transfer warnings, or error -65817 for an input read, -65813 for an invalid vertex index, or -65818 for the write. + * + * A store's first write can need its own working reservation, for example to create a writable copy of a + * read-only imported source. Running that write while the caller holds no reservation keeps it from competing + * with the compute buffers. The stored normals are never read, so an imported output chunk is never decoded here. + * The function reads only the inputs that the block loop reads again; that loop later writes the same value again. + */ +Result<> PublishFirstNormal(const INodeGeometry2D::SharedFaceList::store_type& triangles, const INodeGeometry2D::SharedVertexList::store_type& verts, Float64AbstractDataStore& normals, + MeshingUtilities::detail::TriangleNormalCalculator calculateNormal) +{ + const usize numVertices = verts.getNumberOfTuples(); + std::array face{}; + auto result = triangles.copyIntoBuffer(0, nonstd::span(face.data(), face.size())); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65817, "Error reading the connectivity of triangle 0 before recalculating triangle normals.")); + } + std::array coordinates{}; + for(usize corner = 0; corner < face.size(); ++corner) + { + const auto vertexIndex = face[corner]; + if(vertexIndex >= numVertices) + { + return MergeResults(std::move(result), MakeErrorResult(-65813, fmt::format("Triangle {} references vertex index {}, but the vertex tuple count is {}. Valid vertex indices are less than {}.", 0, + vertexIndex, numVertices, numVertices))); + } + } + for(usize corner = 0; corner < face.size(); ++corner) + { + result = MergeResults(std::move(result), verts.copyIntoBuffer(face[corner] * 3, nonstd::span(coordinates.data() + (corner * 3), 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65817, "Error reading the vertex coordinates of triangle 0 before recalculating triangle normals.")); + } + } + const auto normal = calculateNormal(coordinates.data(), coordinates.data() + 3, coordinates.data() + 6); + const std::array values{normal[0], normal[1], normal[2]}; + result = MergeResults(std::move(result), normals.copyFromBuffer(0, nonstd::span(values.data(), values.size()))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65818, "Error writing the normal of triangle 0 before recalculating triangle normals.")); + } + return result; +} +} // namespace + +Result<> MeshingUtilities::detail::CalculateNormalsInBlocks(const INodeGeometry2D::SharedFaceList::store_type& triangles, const INodeGeometry2D::SharedVertexList::store_type& verts, + Float64AbstractDataStore& normals, const std::atomic_bool& shouldCancel, TriangleNormalCalculator calculateNormal) +{ + const usize numTriangles = triangles.getNumberOfTuples(); + const usize numVertices = verts.getNumberOfTuples(); + if(shouldCancel || numTriangles == 0) + { + return {}; + } + + constexpr usize maximum = std::numeric_limits::max(); + const usize maximumTriangles = std::min(k_NormalBlockTriangles, numTriangles); + if(verts.getNumberOfComponents() != 3 || triangles.getNumberOfComponents() != 3 || numVertices > maximum / (3 * sizeof(float32)) || numTriangles > maximum / 3 || + numVertices * (3 * sizeof(float32)) > maximum - maximumTriangles * k_NormalFastBytesPerFace) + { + return MakeErrorResult(-65814, "Triangle normal buffers require addressable three-component vertex and face arrays. Reduce the mesh size or repair the array shapes."); + } + const usize vertexBytes = numVertices * 3 * sizeof(float32); + const usize desiredBytes = vertexBytes + (maximumTriangles * k_NormalFastBytesPerFace); + auto& budget = CacheMemoryBudgetManager::instance(); + const uint64 maximumWorkingBytes = budget.maximumWorkingMemoryBytes(); + const uint64 reservedBytes = budget.reservedWorkingMemoryBytes(); + const usize observedHeadroom = static_cast(std::min(maximumWorkingBytes > reservedBytes ? maximumWorkingBytes - reservedBytes : 0, maximum)); + const auto observedPlan = planNormalWorkingMemory(triangles, verts, observedHeadroom); + const usize preferredBytes = std::min(desiredBytes, maximumTriangles * k_NormalGatherBytesPerFace); + const uint64 requestedBytes = std::min(observedPlan.FaceCapacity == 0 ? preferredBytes : observedPlan.PayloadBytes, budget.maximumWorkingMemoryBytes()); + // A snapshot avoids known excess admission; a racing reservation can still reduce the actual grant. + auto reservation = budget.reserveWorkingMemory(requestedBytes); + usize grantedBytes = static_cast(std::min(reservation.sizeBytes(), maximum)); + auto plan = planNormalWorkingMemory(triangles, verts, grantedBytes); + if(plan.FaceCapacity == 0) + { + return MakeNormalAdmissionError(-65815, requestedBytes, grantedBytes, vertexBytes); + } + Result<> result; + if(normals.getStoreType() == IDataStore::StoreType::OutOfCore) + { + // A store of type OutOfCore can reserve its own working memory on its first write. Release the admission + // while the normal of triangle 0 is calculated and written, then admit the checked plan again. No buffer + // exists yet, so the released bytes fund nothing. If the new admission fails, triangle 0 keeps its new + // normal and every later triangle keeps its old value. + const usize admittedBytes = plan.PayloadBytes; + reservation = {}; + result = PublishFirstNormal(triangles, verts, normals, calculateNormal); + if(result.invalid()) + { + return result; + } + // Cancellation during the publication stops before the second admission, buffer allocation, or any block transfer. + if(shouldCancel) + { + return result; + } + reservation = budget.reserveWorkingMemory(static_cast(admittedBytes)); + grantedBytes = static_cast(std::min(reservation.sizeBytes(), maximum)); + if(grantedBytes < admittedBytes) + { + plan = planNormalWorkingMemory(triangles, verts, grantedBytes); + if(plan.FaceCapacity == 0) + { + return MergeResults(std::move(result), MakeNormalAdmissionError(-65819, admittedBytes, grantedBytes, vertexBytes)); + } + } + } + const bool cacheVertices = plan.CacheVertices; + const usize bufferTriangles = plan.FaceCapacity; + const usize pageVertices = plan.PageVertices; + // The checked plan fits the actual grant; release any remainder before allocations or I/O. + reservation.shrinkTo(plan.PayloadBytes); + try + { + std::vector vertices(cacheVertices ? numVertices * 3 : 0); + if(cacheVertices) + { + result = MergeResults(std::move(result), verts.copyIntoBuffer(0, {vertices.data(), vertices.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65810, "Error reading vertex coordinates for triangle normal calculation.")); + } + } + std::vector order(cacheVertices ? 0 : bufferTriangles * 3); + std::vector gathered(cacheVertices ? 0 : bufferTriangles * 9); + std::vector page(pageVertices * 3); + std::vector triangleBuffer(bufferTriangles * 3); + std::vector normalBuffer(bufferTriangles * 3); + for(usize start = 0; start < numTriangles;) + { + if(shouldCancel) + { + return result; + } + const usize count = std::min(bufferTriangles, numTriangles - start); + result = MergeResults(std::move(result), triangles.copyIntoBuffer(start * 3, nonstd::span(triangleBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65811, fmt::format("Error reading connectivity for triangle normal calculation at triangle {} ({} triangles).", start, count))); + } + + for(usize triangle = 0; triangle < count; ++triangle) + { + const usize index = triangle * 3; + for(usize vertex = 0; vertex < 3; ++vertex) + { + const auto vertexIndex = triangleBuffer[index + vertex]; + if(vertexIndex >= numVertices) + { + return MergeResults(std::move(result), + MakeErrorResult(-65813, fmt::format("Triangle {} references vertex index {}, but the vertex tuple count is {}. Valid vertex indices are less than {}.", + start + triangle, vertexIndex, numVertices, numVertices))); + } + } + } + if(!cacheVertices) + { + result = + MergeResults(std::move(result), GatherVertexCoordinates(verts, {triangleBuffer.data(), count * 3}, {order.data(), count * 3}, {gathered.data(), count * 9}, {page.data(), page.size()})); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65810, fmt::format("Error reading vertex coordinates for triangle normal calculation at triangle {} ({} triangles).", start, count))); + } + } + for(usize triangle = 0; triangle < count; ++triangle) + { + const usize index = triangle * 3; + const nonstd::span coordinates = + cacheVertices ? nonstd::span(vertices.data(), vertices.size()) : nonstd::span(gathered.data(), gathered.size()).subspan(triangle * 9, 9); + const usize firstVertex = cacheVertices ? triangleBuffer[index] * 3 : 0; + const usize secondVertex = cacheVertices ? triangleBuffer[index + 1] * 3 : 3; + const usize thirdVertex = cacheVertices ? triangleBuffer[index + 2] * 3 : 6; + const auto normal = calculateNormal(coordinates.subspan(firstVertex, 3).data(), coordinates.subspan(secondVertex, 3).data(), coordinates.subspan(thirdVertex, 3).data()); + normalBuffer[index] = normal[0]; + normalBuffer[index + 1] = normal[1]; + normalBuffer[index + 2] = normal[2]; + } + + result = MergeResults(std::move(result), normals.copyFromBuffer(start * 3, nonstd::span(normalBuffer.data(), count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65812, fmt::format("Error writing normals at triangle {} ({} triangles).", start, count))); + } + start += count; + } + return result; + } catch(const std::bad_alloc&) + { + return MergeResults(std::move(result), MakeErrorResult(-65816, "Triangle normal calculation could not allocate its admitted working buffers. Reduce the mesh size or free memory.")); + } catch(const std::length_error&) + { + return MergeResults(std::move(result), MakeErrorResult(-65816, "Triangle normal calculation exceeded the supported buffer allocation size. Reduce the mesh size.")); + } +} + +Result<> MeshingUtilities::CalculateNormals(const INodeGeometry2D::SharedFaceList& triangles, const INodeGeometry2D::SharedVertexList& verts, Float64AbstractDataStore& normals, + const std::atomic_bool& shouldCancel) +{ + // The output is a borrowed store rather than a DataArray, so include its residency explicitly. + const bool usesOutOfCoreStore = AnyOutOfCore({&triangles, &verts}) || normals.getStoreType() == IDataStore::StoreType::OutOfCore; + const bool useOutOfCoreAlgorithm = !ForceInCoreAlgorithm() && (usesOutOfCoreStore || ForceOocAlgorithm()); + RecordAlgorithmPathExecution(useOutOfCoreAlgorithm ? AlgorithmPath::OutOfCore : AlgorithmPath::InCore, usesOutOfCoreStore); + if(useOutOfCoreAlgorithm) + { + return detail::CalculateNormalsInBlocks(triangles.getDataStoreRef(), verts.getDataStoreRef(), normals, shouldCancel, + [](const float32* a, const float32* b, const float32* c) -> std::array { + const Eigen::Vector3d vertA = Eigen::Vector3d{a[0], a[1], a[2]}; + const Eigen::Vector3d vertB = Eigen::Vector3d{b[0], b[1], b[2]}; + const Eigen::Vector3d vertC = Eigen::Vector3d{c[0], c[1], c[2]}; + const Eigen::Vector3d vecA = vertB - vertA; + const Eigen::Vector3d vecB = vertC - vertA; + Eigen::Vector3d normal = vecA.cross(vecB); + normal.normalize(); + return {normal[0], normal[1], normal[2]}; + }); + } + + ParallelDataAlgorithm dataAlg; + dataAlg.setRange(0ULL, triangles.getNumberOfTuples()); + dataAlg.execute(CalculateNormalsImpl(triangles.getDataStoreRef(), verts.getDataStoreRef(), normals, shouldCancel)); + return {}; +} + Result<> MeshingUtilities::MakeEmptyMeshWarning(const DataPath& triangleGeomPath, usize numCells, usize numVertices) { return MakeWarningVoidResult(k_EmptyMeshAfterSkinRemovalWarning, diff --git a/src/simplnx/Utilities/Meshing/TriangleUtilities.hpp b/src/simplnx/Utilities/Meshing/TriangleUtilities.hpp index d88202d16d..e2f91c719b 100644 --- a/src/simplnx/Utilities/Meshing/TriangleUtilities.hpp +++ b/src/simplnx/Utilities/Meshing/TriangleUtilities.hpp @@ -115,6 +115,27 @@ inline static constexpr usize k_22 = 8; * @return Signed volume contribution. */ SIMPLNX_EXPORT INodeGeometry2D::SharedVertexList::value_type FindTriangleVolume(const std::array& vertIndices, const INodeGeometry2D::SharedVertexList::store_type& vertices); + +using TriangleNormalCalculator = std::array (*)(const float32* vertA, const float32* vertB, const float32* vertC); + +/** + * @brief Computes normals with serial block I/O and admitted coordinate buffers. + * @param triangles Provides three vertex indexes per triangle. + * @param verts Provides XYZ coordinates, cached when admitted or gathered through bounded pages. + * @param normals Receives three normal components per triangle. + * @param shouldCancel Stops before loading vertices or processing the next block. + * @param calculateNormal Preserves the caller's arithmetic using three resident XYZ pointers. + * @return Accumulated I/O warnings and errors, or success after cancellation. + * + * When the output store type is OutOfCore, the function calculates the normal of triangle 0 and writes it + * before it retains the compute admission. A store's first write can reserve its own working memory, for example + * to create a writable copy of a read-only imported source, and must not compete with the compute buffers. The + * stored normals are never read. Error -65817 reports an input read failure for triangle 0, error -65813 an + * invalid vertex index, and error -65818 the write. Error -65819 reports a grant lost to another consumer while + * the admission was released; triangle 0 then holds its new normal and later triangles keep their old values. + */ +SIMPLNX_EXPORT Result<> CalculateNormalsInBlocks(const INodeGeometry2D::SharedFaceList::store_type& triangles, const INodeGeometry2D::SharedVertexList::store_type& verts, + Float64AbstractDataStore& normals, const std::atomic_bool& shouldCancel, TriangleNormalCalculator calculateNormal); } // namespace detail /** @@ -190,6 +211,17 @@ class SIMPLNX_EXPORT CalculateNormalsImpl const std::atomic_bool& m_ShouldCancel; }; +/** + * @brief Computes Eigen double-precision normals, using block I/O when any store is out of core. + * @param triangles Provides triangle connectivity. + * @param verts Provides vertex coordinates. + * @param normals Receives three values per triangle. + * @param shouldCancel Stops before later triangles or OOC blocks. + * @return Accumulated I/O warnings and errors, or success after cancellation. + */ +SIMPLNX_EXPORT Result<> CalculateNormals(const INodeGeometry2D::SharedFaceList& triangles, const INodeGeometry2D::SharedVertexList& verts, Float64AbstractDataStore& normals, + const std::atomic_bool& shouldCancel); + /** * @brief Accumulates signed triangle volume contributions by feature ID. * @tparam ContainerT Specifies a random-access volume container. diff --git a/src/simplnx/Utilities/Meshing/VertexUtilities.cpp b/src/simplnx/Utilities/Meshing/VertexUtilities.cpp index 28a5dcb1f9..647a64465a 100644 --- a/src/simplnx/Utilities/Meshing/VertexUtilities.cpp +++ b/src/simplnx/Utilities/Meshing/VertexUtilities.cpp @@ -1,9 +1,95 @@ #include "VertexUtilities.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" +#include "simplnx/DataStructure/Geometry/IGeometry.hpp" +#include "simplnx/DataStructure/IO/Generic/ITemporaryRecordStore.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include + using namespace nx::core; namespace { +Result<> readCoordinatePage(const Float32AbstractDataStore& vertices, usize first, nonstd::span page) +{ + return vertices.copyIntoBuffer(first * 3, page); +} + +Result<> readCoordinatePage(const ITemporaryRecordStore& vertices, usize first, nonstd::span page) +{ + const std::atomic_bool active{false}; + const usize count = page.size() / 3; + const auto bytes = std::as_writable_bytes(std::span(page.data(), page.size())); + auto read = vertices.read(first, count, {bytes.data(), bytes.size()}, active); + if(read.invalid()) + { + return ConvertResult(std::move(read)); + } + if(read.value() != count) + { + return MergeResults(ConvertResult(std::move(read)), MakeErrorResult(-65901, "Temporary vertex coordinates returned fewer XYZ records than requested.")); + } + return ConvertResult(std::move(read)); +} + +template +Result<> gatherCoordinatePages(const Store& vertices, usize vertexCount, nonstd::span indexes, nonstd::span order, nonstd::span gathered, + nonstd::span page) +{ + constexpr usize maximum = std::numeric_limits::max(); + if(vertexCount > maximum / 3 || indexes.size() > maximum / 3 || order.size() < indexes.size() || gathered.size() < indexes.size() * 3 || page.size() % 3 != 0 || (!indexes.empty() && page.empty())) + { + return MakeErrorResult(-65902, "Vertex-coordinate gather received invalid buffer sizes. Supply one order entry and three coordinates per vertex, and a whole-XYZ read page."); + } + for(usize request = 0; request < indexes.size(); ++request) + { + if(indexes[request] >= vertexCount) + { + return MakeErrorResult(-65903, fmt::format("Vertex-coordinate request {} references vertex {}, but the vertex count is {}.", request, indexes[request], vertexCount)); + } + order[request] = request; + } + if(indexes.empty()) + { + return {}; + } + const auto requests = order.first(indexes.size()); + std::ranges::sort(requests, [&indexes](usize left, usize right) { return indexes[left] < indexes[right]; }); + const usize capacity = page.size() / 3; + usize loadedPage = maximum; + Result<> result; + for(usize request = 0; request < indexes.size(); ++request) + { + const usize ordinal = order[request]; + const usize vertex = static_cast(indexes[ordinal]); + const usize first = (vertex / capacity) * capacity; + if(first != loadedPage) + { + const usize count = std::min(capacity, vertexCount - first); + result = MergeResults(std::move(result), readCoordinatePage(vertices, first, page.first(count * 3))); + if(result.invalid()) + { + return MergeResults(std::move(result), MakeErrorResult(-65904, fmt::format("Error reading vertex-coordinate page at vertex {} ({} vertices).", first, count))); + } + loadedPage = first; + } + const auto coordinates = page.subspan((vertex - first) * 3, 3); + const auto destination = gathered.subspan(ordinal * 3, 3); + std::ranges::copy(coordinates, destination.begin()); + } + return result; +} + // Uses Hoare's method for speed IGeometry::MeshIndexType ProcessSection(std::vector& sorted, IGeometry::MeshIndexType begin, IGeometry::MeshIndexType end, const INodeGeometry0D::SharedVertexList::store_type& vertices, IGeometry::MeshIndexType offset) @@ -52,6 +138,26 @@ void QuickSortVertices(std::vector& sorted, IGeometry: } } // namespace +Result<> MeshingUtilities::detail::GatherVertexCoordinates(const Float32AbstractDataStore& vertices, nonstd::span indexes, nonstd::span order, + nonstd::span gathered, nonstd::span page) +{ + if(vertices.getNumberOfComponents() != 3) + { + return MakeErrorResult(-65900, "Vertex-coordinate gather requires three-component XYZ tuples."); + } + return gatherCoordinatePages(vertices, vertices.getNumberOfTuples(), indexes, order, gathered, page); +} + +Result<> MeshingUtilities::detail::GatherVertexCoordinates(const ITemporaryRecordStore& vertices, nonstd::span indexes, nonstd::span order, + nonstd::span gathered, nonstd::span page) +{ + if(vertices.recordSize() != 3 * sizeof(float32) || vertices.recordCount() > std::numeric_limits::max() || page.size() / 3 > vertices.maxRecordsPerBatch()) + { + return MakeErrorResult(-65905, "Temporary vertex-coordinate gather requires addressable XYZ records and a page no larger than the provider batch limit."); + } + return gatherCoordinatePages(vertices, static_cast(vertices.recordCount()), indexes, order, gathered, page); +} + MeshingUtilities::SortedVerticesList MeshingUtilities::OrderSharedVertices(const nx::core::INodeGeometry0D& geom, const std::atomic_bool& shouldCancel) { const BoundingBox3Df& bounds = geom.getBoundingBox(); diff --git a/src/simplnx/Utilities/Meshing/VertexUtilities.hpp b/src/simplnx/Utilities/Meshing/VertexUtilities.hpp index 58c689362d..53922be15f 100644 --- a/src/simplnx/Utilities/Meshing/VertexUtilities.hpp +++ b/src/simplnx/Utilities/Meshing/VertexUtilities.hpp @@ -1,11 +1,52 @@ #pragma once #include "simplnx/Common/Result.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" #include "simplnx/DataStructure/Geometry/IGeometry.hpp" #include "simplnx/DataStructure/Geometry/INodeGeometry0D.hpp" +#include + +#include +#include + +namespace nx::core +{ +class ITemporaryRecordStore; +} + namespace nx::core::MeshingUtilities { +namespace detail +{ +/** + * @brief Reads requested XYZ tuples through caller-owned bounded buffers. + * @param vertices Supplies three-component coordinates. + * @param indexes Specifies vertex tuples in the required output order. + * @param order Supplies one permutation element per request. + * @param gathered Receives three coordinates per request, in the original order. + * @param page Supplies a nonempty whole-XYZ read page when requests exist. + * @return Shape, index, capacity, or read error, or success with backend warnings. + * @note Sorts only read requests. The caller owns admission and cancellation checkpoints. + */ +SIMPLNX_EXPORT Result<> GatherVertexCoordinates(const Float32AbstractDataStore& vertices, nonstd::span indexes, nonstd::span order, + nonstd::span gathered, nonstd::span page); + +/** + * @brief Reads temporary XYZ records through caller-owned bounded buffers. + * @param vertices Supplies records of three float32 coordinates each. + * @param indexes Specifies vertex records in the required output order. + * @param order Supplies one permutation element per request. + * @param gathered Receives three coordinates per request, in the original order. + * @param page Supplies a whole-XYZ read page within the provider batch limit; nonempty when requests exist. + * @return Record shape, index, capacity, or read error, or success with backend warnings. + * @note Sorts only read requests. The caller owns admission and cancellation checkpoints. + */ +SIMPLNX_EXPORT Result<> GatherVertexCoordinates(const ITemporaryRecordStore& vertices, nonstd::span indexes, nonstd::span order, nonstd::span gathered, + nonstd::span page); +} // namespace detail + enum SIMPLNX_EXPORT AxialAlignment : uint8 { X = 0, diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp index ce44c7fcfe..a2d1174dc9 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.cpp @@ -1,5 +1,7 @@ #include "Dream3dIO.hpp" #include "Dream3dIOInternal.hpp" +#include "simplnx/Common/Result.hpp" +#include #include "simplnx/Common/Aliases.hpp" #include "simplnx/Core/Application.hpp" @@ -167,15 +169,24 @@ void WriteGeomXdmf(std::ostream& out, const ImageGeom& imageGeom, std::string_vi std::array volDims = {static_cast(dims.getX()), static_cast(dims.getY()), static_cast(dims.getZ())}; - out << " " << "\n"; - out << " )" << "\n"; - out << R"( " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << R"( )" << origin[2] << " " << origin[1] << " " << origin[0] << "" << "\n"; - out << " " << "\n"; - out << R"( )" << spacing[2] << " " << spacing[1] << " " << spacing[0] << "" << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " )" + << "\n"; + out << R"( " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << R"( )" << origin[2] << " " << origin[1] << " " << origin[0] << "" + << "\n"; + out << " " + << "\n"; + out << R"( )" << spacing[2] << " " << spacing[1] << " " << spacing[0] << "" + << "\n"; + out << " " + << "\n"; } void WriteGeomXdmf(std::ostream& out, const RectGridGeom& rectGridGeom, std::string_view hdf5FilePath) @@ -196,20 +207,31 @@ void WriteGeomXdmf(std::ostream& out, const RectGridGeom& rectGridGeom, std::str std::array volDims = {static_cast(dims.getX()), static_cast(dims.getY()), static_cast(dims.getZ())}; - out << " " << "\n"; - out << " )" << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " getNumberOfTuples() << "\" NumberType=\"Float\" Precision=\"4\">" << "\n"; + out << " " + << "\n"; + out << " )" + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " getNumberOfTuples() << "\" NumberType=\"Float\" Precision=\"4\">" + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << xBoundsPath.toString() << "\n"; - out << " " << "\n"; - out << " getNumberOfTuples() << "\" NumberType=\"Float\" Precision=\"4\">" << "\n"; + out << " " + << "\n"; + out << " getNumberOfTuples() << "\" NumberType=\"Float\" Precision=\"4\">" + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << yBoundsPath.toString() << "\n"; - out << " " << "\n"; - out << " getNumberOfTuples() << "\" NumberType=\"Float\" Precision=\"4\">" << "\n"; + out << " " + << "\n"; + out << " getNumberOfTuples() << "\" NumberType=\"Float\" Precision=\"4\">" + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << zBoundsPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; } void WriteGeomXdmf(std::ostream& out, const VertexGeom& vertexGeom, std::string_view hdf5FilePath) @@ -224,21 +246,33 @@ void WriteGeomXdmf(std::ostream& out, const VertexGeom& vertexGeom, std::string_ DataPath geomPath = vertexGeom.getDataPaths().at(0); - out << " " << "\n"; - out << " )" << "\n"; - - out << R"( " << "\n"; - out << R"( " << "\n"; - out << " " << hdf5FilePath << ":/DataStructure/" << geomPath.toString() << "/_VertexIndices" << "\n"; - out << " " << "\n"; - out << " " << "\n"; - - out << " " << "\n"; - out << R"( )" << "\n"; + out << " " + << "\n"; + out << " )" + << "\n"; + + out << R"( " + << "\n"; + out << R"( " + << "\n"; + out << " " << hdf5FilePath << ":/DataStructure/" << geomPath.toString() << "/_VertexIndices" + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + + out << " " + << "\n"; + out << R"( )" + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << verticesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << "" << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << "" + << "\n"; } void WriteGeomXdmf(std::ostream& out, const EdgeGeom& edgeGeom, std::string_view hdf5FilePath) @@ -254,19 +288,30 @@ void WriteGeomXdmf(std::ostream& out, const EdgeGeom& edgeGeom, std::string_view DataPath edgesPath = edgeGeom.getEdgesRef().getDataPaths().at(0); DataPath verticesPath = edgeGeom.getVerticesRef().getDataPaths().at(0); - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << edgesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << verticesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << "" << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << "" + << "\n"; } void WriteGeomXdmf(std::ostream& out, const TriangleGeom& triangleGeom, std::string_view hdf5FilePath) @@ -282,19 +327,30 @@ void WriteGeomXdmf(std::ostream& out, const TriangleGeom& triangleGeom, std::str DataPath facesPath = triangleGeom.getFacesRef().getDataPaths().at(0); DataPath verticesPath = triangleGeom.getVerticesRef().getDataPaths().at(0); - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << facesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << verticesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << "" << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << "" + << "\n"; } void WriteGeomXdmf(std::ostream& out, const QuadGeom& quadGeom, std::string_view hdf5FilePath) @@ -309,19 +365,30 @@ void WriteGeomXdmf(std::ostream& out, const QuadGeom& quadGeom, std::string_view DataPath facesPath = quadGeom.getFacesRef().getDataPaths().at(0); DataPath verticesPath = quadGeom.getVerticesRef().getDataPaths().at(0); - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << facesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << verticesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << "" << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << "" + << "\n"; } void WriteGeomXdmf(std::ostream& out, const TetrahedralGeom& tetrahedralGeom, std::string_view hdf5FilePath) @@ -336,19 +403,30 @@ void WriteGeomXdmf(std::ostream& out, const TetrahedralGeom& tetrahedralGeom, st DataPath polyhedraPath = tetrahedralGeom.getPolyhedraRef().getDataPaths().at(0); DataPath verticesPath = tetrahedralGeom.getVerticesRef().getDataPaths().at(0); - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << polyhedraPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << verticesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << "" << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << "" + << "\n"; } void WriteGeomXdmf(std::ostream& out, const HexahedralGeom& hexhedralGeom, std::string_view hdf5FilePath) @@ -363,33 +441,50 @@ void WriteGeomXdmf(std::ostream& out, const HexahedralGeom& hexhedralGeom, std:: DataPath polyhedraPath = hexhedralGeom.getPolyhedraRef().getDataPaths().at(0); DataPath verticesPath = hexhedralGeom.getVerticesRef().getDataPaths().at(0); - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << polyhedraPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; out << " " << hdf5FilePath << ":/DataStructure/" << verticesPath.toString() << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << "" << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << "" + << "\n"; } void WriteXdmfHeader(std::ostream& out) { - out << "" << "\n"; - out << "" << "\n"; - out << "" << "\n"; - out << " " << "\n"; + out << "" + << "\n"; + out << "" + << "\n"; + out << "" + << "\n"; + out << " " + << "\n"; } void WriteXdmfFooter(std::ostream& xdmf) { - xdmf << " " << "\n"; - xdmf << "" << "\n"; + xdmf << " " + << "\n"; + xdmf << "" + << "\n"; } std::string GetXdmfArrayType(usize numComp) @@ -437,13 +532,18 @@ void WriteXdmfAttributeDataHelper(std::ostream& out, usize numComp, std::string_ { out << " " << "\n"; + out << "Center=\"" << centering << "\">" + << "\n"; // Open the Tag out << R"( " << "\n"; + out << "NumberType=\"" << xdmfTypeName << "\" " + << "Precision=\"" << precision << "\" >" + << "\n"; out << " " << hdf5DatasetPath << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; } else if(numComp == 2 || numComp == 6) { @@ -451,48 +551,77 @@ void WriteXdmfAttributeDataHelper(std::ostream& out, usize numComp, std::string_ out << " " << "\n"; + out << "Center=\"" << centering << "\">" + << "\n"; // Open the Tag out << R"( " << "\n"; - out << " " << "\n"; - out << " 0 0" << "\n"; - out << " 1 1" << "\n"; - out << " " << dimStrHalf << " " << "\n"; + out << "Type=\"HyperSlab\" " + << "Name=\"" << arrayName << " (Feature 0)\" >" + << "\n"; + out << " " + << "\n"; + out << " 0 0" + << "\n"; + out << " 1 1" + << "\n"; + out << " " << dimStrHalf << " " + << "\n"; out << "\n"; - out << R"( " << "\n"; + out << R"( " + << "\n"; out << " " << hdf5DatasetPath << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n" + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n" << "\n"; // Second Slab out << " " << "\n"; + out << "Center=\"" << centering << "\">" + << "\n"; // Open the Tag out << R"( " << "\n"; - out << " " << "\n"; + out << "Type=\"HyperSlab\" " + << "Name=\"" << arrayName << " (Feature 1)\" >" + << "\n"; + out << " " + << "\n"; out << " 0 " << (numComp / 2) << "\n"; - out << " 1 1" << "\n"; - out << " " << dimStrHalf << " " << "\n"; + out << " 1 1" + << "\n"; + out << " " << dimStrHalf << " " + << "\n"; out << "\n"; - out << R"( " << "\n"; + out << R"( " + << "\n"; out << " " << hdf5DatasetPath << "\n"; - out << " " << "\n"; - out << " " << "\n"; - out << " " << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; + out << " " + << "\n"; } } void WriteXdmfGeomFooter(std::ostream& xdmf, std::string_view geomName) { - xdmf << " " << "\n"; - xdmf << " " << "\n"; + xdmf << " " + << "\n"; + xdmf << " " + << "\n"; } void WriteXdmfAttributeMatrix(std::ostream& out, const AttributeMatrix& attributeMatrix, std::string_view geomName, std::string_view hdf5FilePath, std::string_view centering) @@ -2349,9 +2478,10 @@ Result<> FinishImportingLegacyDataObject(DataStructure& dataStructure, const nx: } } -Result ImportLegacyDataStructure(const nx::core::HDF5::FileIO& fileReader, bool preflight) +Result ImportLegacyDataStructure(const nx::core::HDF5::FileIO& fileReader, bool preflight, std::shared_ptr resolver) { DataStructure dataStructure; + dataStructure.setFormatResolver(std::move(resolver)); auto dcaGroup = fileReader.openGroup(k_LegacyDataStructureGroupTag); @@ -2368,6 +2498,26 @@ Result ImportLegacyDataStructure(const nx::core::HDF5::FileIO& fi return nx::core::ConvertResultTo(std::move(result), std::move(dataStructure)); } +Result ImportLegacyDataStructure(const nx::core::HDF5::FileIO& fileReader, bool preflight) +{ + return ImportLegacyDataStructure(fileReader, preflight, {}); +} + +Result DREAM3D::detail::ImportMetadata(const nx::core::HDF5::FileIO& fileReader, std::shared_ptr resolver) +{ + const auto version = GetFileVersion(fileReader); + if(version == k_CurrentFileVersion) + { + return HDF5::DataStructureReader::ReadFile(fileReader, true, std::move(resolver)); + } + if(version == k_LegacyFileVersion) + { + return ImportLegacyDataStructure(fileReader, true, std::move(resolver)); + } + return MakeErrorResult(k_InvalidDataStructureVersion, fmt::format("Cannot import metadata from '{}': file version '{}' is not supported. Expected '{}' or '{}'.", + fileReader.getFilePath().string(), version, k_CurrentFileVersion, k_LegacyFileVersion)); +} + Result DREAM3D::ImportDataStructureFromFile(const nx::core::HDF5::FileIO& fileReader, bool preflight) { const auto fileVersion = GetFileVersion(fileReader); @@ -2754,7 +2904,7 @@ Result<> DREAM3D::WriteFile(const std::filesystem::path& path, const DataStructu WriteXdmf(xdmfFilePath, dataStructure, path.filename().string()); } - return {}; + return result; } Result<> DREAM3D::WriteRecoveryFile(const fs::path& path, const DataStructure& dataStructure, const Pipeline& pipeline, std::optional userDataFilePath) diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIOInternal.hpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIOInternal.hpp index f662784dde..ccf4454363 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIOInternal.hpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dIOInternal.hpp @@ -4,11 +4,13 @@ #include "simplnx/DataStructure/DataPath.hpp" #include "simplnx/simplnx_export.hpp" +#include #include namespace nx::core { class DataStructure; +class IDataStoreFormatResolver; namespace HDF5 { @@ -17,6 +19,15 @@ class FileIO; namespace DREAM3D::detail { +/** + * @brief Imports metadata using one local policy throughout NX or legacy traversal. + * @param fileReader Supplies the open file and its version. + * @param resolver Supplies policy before any + * metadata factory runs. + * @return Metadata and original disk-import diagnostics. + */ +SIMPLNX_EXPORT Result ImportMetadata(const HDF5::FileIO& fileReader, std::shared_ptr resolver); + /** * @brief Materializes exact imported leaf paths from one open DREAM3D file. * @param preparedStructure Owns placeholders whose final plans are already recorded. diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.cpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.cpp index 1a0d869ae1..30011428bb 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.cpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.cpp @@ -1,9 +1,13 @@ #include "Dream3dPreflightCache.hpp" +#include "simplnx/Common/Result.hpp" +#include +#include #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/EmptyStringStore.hpp" #include "simplnx/DataStructure/IDataArray.hpp" #include "simplnx/DataStructure/INeighborList.hpp" +#include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" #include "simplnx/DataStructure/NeighborList.hpp" #include "simplnx/DataStructure/StringArray.hpp" #include "simplnx/DataStructure/StringStore.hpp" @@ -11,10 +15,11 @@ #include "simplnx/Utilities/DataStoreUtilities.hpp" #include "simplnx/Utilities/FilterUtilities.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" +#include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIOInternal.hpp" #include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" #include "simplnx/Utilities/StoreCopyUtilities.hpp" -#include +#include #include #include @@ -90,16 +95,24 @@ std::string MakeKey(const fs::path& filePath) /** * @brief Imports one metadata-only DataStructure from disk. * @param filePath Identifies the DREAM3D file. - * @return Imported structure or the standard open or import error. + * @param neutral Selects local resident metadata policy. + * @return Imported structure, source version, and disk diagnostics. */ -Result ReadFromDisk(const fs::path& filePath) +Result ReadFromDisk(const fs::path& filePath, bool neutral) { auto fileReader = nx::core::HDF5::FileIO::ReadFile(filePath); if(!fileReader.isValid()) { - return MakeErrorResult(k_FailedOpenFileIOError, fmt::format("Failed to open the HDF5 file at the specified path: '{}'", filePath.string())); + return MakeErrorResult(k_FailedOpenFileIOError, fmt::format("Failed to open the HDF5 file at the specified path: '{}'", filePath.string())); } - return ImportDataStructureFromFile(fileReader, true); + auto version = GetFileVersion(fileReader); + auto result = neutral ? detail::ImportMetadata(fileReader, std::make_shared()) : ImportDataStructureFromFile(fileReader, true); + if(result.invalid()) + { + return ConvertInvalidResult(std::move(result)); + } + NeutralMetadataHandout handout{std::move(result.value()), std::move(version)}; + return ConvertResultTo(ConvertResult(std::move(result)), std::move(handout)); } /** @@ -272,23 +285,24 @@ Result<> Dream3dPreflightCache::RefreshStores(DataStructure& dataStructure) return result; } -Result Dream3dPreflightCache::PrepareHandout(DataStructure handout, const fs::path& filePath) +Result Dream3dPreflightCache::PrepareHandout(NeutralMetadataHandout handout, const fs::path& filePath) { - auto refreshResult = RefreshStores(handout); + auto refreshResult = RefreshStores(handout.dataStructure); if(refreshResult.invalid()) { for(auto& error : refreshResult.errors()) { error.message = fmt::format("Cannot prepare cached metadata handout for file '{}': {}", filePath.string(), error.message); } - return ConvertInvalidResult(std::move(refreshResult)); + return ConvertInvalidResult(std::move(refreshResult)); } - return ConvertResultTo(std::move(refreshResult), std::move(handout)); + return ConvertResultTo(std::move(refreshResult), std::move(handout)); } -Result> Dream3dPreflightCache::tryServeFromCache(const std::string& key, uint64 fileSize, const fs::file_time_type& mtime, const fs::path& filePath) +Result> Dream3dPreflightCache::tryServeFromCache(const CacheKey& key, uint64 fileSize, const fs::file_time_type& mtime, const fs::path& filePath) { - std::optional handout; + std::optional handout; + std::vector diskWarnings; { const std::lock_guard lock(m_Mutex); auto iter = m_Entries.find(key); @@ -296,129 +310,113 @@ Result> Dream3dPreflightCache::tryServeFromCache(co { m_Hits++; iter->second.lastUsedTick = ++m_Tick; - // Copy while the entry cannot be evicted. Ordinary bulk datasets remain placeholders; - // small eager Statistics stores are shared only until handout preparation below. handout = iter->second.master; + diskWarnings = iter->second.diskWarnings; } } - if(handout.has_value()) + if(!handout.has_value()) { - // Shared pointers keep source stores alive after the table lock releases. - // Masters are immutable, so store isolation needs no table synchronization. - auto preparedResult = PrepareHandout(std::move(*handout), filePath); - if(preparedResult.invalid()) - { - return ConvertInvalidResult>(std::move(preparedResult)); - } - Result> result{std::optional{std::move(preparedResult.value())}}; - result.warnings() = std::move(preparedResult.warnings()); - return result; + return {std::nullopt}; + } + // Shared owners keep stores alive after the table lock releases. Isolation never changes the immutable master. + auto prepared = PrepareHandout(std::move(*handout), filePath); + PrependWarnings(prepared, diskWarnings); + if(prepared.invalid()) + { + return ConvertInvalidResult>(std::move(prepared)); } - return {std::nullopt}; + std::optional value{std::move(prepared.value())}; + return ConvertResultTo>(ConvertResult(std::move(prepared)), std::move(value)); } Result Dream3dPreflightCache::fetch(const fs::path& filePath) { - std::error_code metadataError; - const auto metadata = ReadFileMetadata(filePath, metadataError); - if(!metadata.has_value()) + auto result = fetchMetadata(filePath, false); + if(result.invalid()) { - // A direct serialized read preserves the standard open error. Failed stat - // paths never enter the cache, but their stores still receive current handout policy. - m_Misses++; - const std::lock_guard readLock(m_ReadMutex); - auto diskResult = ReadFromDisk(filePath); - if(diskResult.invalid()) - { - return diskResult; - } - auto diskWarnings = std::move(diskResult.warnings()); - auto preparedResult = PrepareHandout(std::move(diskResult.value()), filePath); - PrependWarnings(preparedResult, diskWarnings); - return preparedResult; + return ConvertInvalidResult(std::move(result)); } - const uint64 fileSize = metadata->fileSize; - const auto mtime = metadata->mtime; + auto structure = std::move(result.value().dataStructure); + return ConvertResultTo(ConvertResult(std::move(result)), std::move(structure)); +} + +Result Dream3dPreflightCache::fetchNeutralMetadata(const fs::path& filePath) +{ + return fetchMetadata(filePath, true); +} - // A recent same-size rewrite can hide inside network timestamp rounding. - // Bypass the cache during the trust window. - if(fs::file_time_type::clock::now() - mtime < k_MtimeTrustWindow) +Result Dream3dPreflightCache::fetchMetadata(const fs::path& filePath, bool neutral) +{ + std::error_code metadataError; + const auto metadata = ReadFileMetadata(filePath, metadataError); + if(!metadata.has_value() || fs::file_time_type::clock::now() - metadata->mtime < k_MtimeTrustWindow) { - // Serialize its HDF5 traversal, then apply current handout policy. + // Stat failures and recent files bypass both modes without changing their policy or diagnostic contracts. m_Misses++; const std::lock_guard readLock(m_ReadMutex); - auto diskResult = ReadFromDisk(filePath); - if(diskResult.invalid()) + auto disk = ReadFromDisk(filePath, neutral); + if(disk.invalid()) { - return diskResult; + return disk; } - auto diskWarnings = std::move(diskResult.warnings()); - auto preparedResult = PrepareHandout(std::move(diskResult.value()), filePath); - PrependWarnings(preparedResult, diskWarnings); - return preparedResult; + auto warnings = std::move(disk.warnings()); + auto result = PrepareHandout(std::move(disk.value()), filePath); + PrependWarnings(result, warnings); + return result; } - const std::string key = MakeKey(filePath); - - // A hit uses only the table mutex and does not wait for an unrelated disk read. - auto hit = tryServeFromCache(key, fileSize, mtime, filePath); + const CacheKey key{MakeKey(filePath), neutral}; + const auto serve = [&]() { return tryServeFromCache(key, metadata->fileSize, metadata->mtime, filePath); }; + auto hit = serve(); if(hit.invalid()) { - return ConvertInvalidResult(std::move(hit)); + return ConvertInvalidResult(std::move(hit)); } if(hit.value().has_value()) { - Result result{std::move(*hit.value())}; - result.warnings() = std::move(hit.warnings()); - return result; + auto value = std::move(*hit.value()); + return ConvertResultTo(ConvertResult(std::move(hit)), std::move(value)); } - // Serialize the complete HDF5 import. This lock precedes every later table lock. + // The read lock precedes any later table lock. Recheck after waiting for another disk import. const std::lock_guard readLock(m_ReadMutex); - - // Another thread can populate the entry while this thread waits for the read lock. - hit = tryServeFromCache(key, fileSize, mtime, filePath); + hit = serve(); if(hit.invalid()) { - return ConvertInvalidResult(std::move(hit)); + return ConvertInvalidResult(std::move(hit)); } if(hit.value().has_value()) { - Result result{std::move(*hit.value())}; - result.warnings() = std::move(hit.warnings()); - return result; + auto value = std::move(*hit.value()); + return ConvertResultTo(ConvertResult(std::move(hit)), std::move(value)); } - m_Misses++; - Result diskResult = ReadFromDisk(filePath); - if(diskResult.invalid()) + auto disk = ReadFromDisk(filePath, neutral); + if(disk.invalid()) { - // A failed import must be retried by the next fetch. - return diskResult; + return disk; } - auto diskWarnings = std::move(diskResult.warnings()); - - DataStructure handout; + auto warnings = std::move(disk.warnings()); + NeutralMetadataHandout handout; { const std::lock_guard lock(m_Mutex); Entry& entry = m_Entries[key]; - entry.master = std::move(diskResult.value()); - entry.fileSize = fileSize; - entry.mtime = mtime; + entry.master = std::move(disk.value()); + // Ordinary warm hits retain their existing behavior. Neutral hits replay only original disk warnings. + entry.diskWarnings = neutral ? warnings : std::vector{}; + entry.fileSize = metadata->fileSize; + entry.mtime = metadata->mtime; entry.lastUsedTick = ++m_Tick; - - // Evict the least-recently-used entry to bound long-session bookkeeping. while(m_Entries.size() > k_Capacity) { auto victim = std::min_element(m_Entries.begin(), m_Entries.end(), [](const auto& a, const auto& b) { return a.second.lastUsedTick < b.second.lastUsedTick; }); m_Entries.erase(victim); } - handout = entry.master; } - auto preparedResult = PrepareHandout(std::move(handout), filePath); - PrependWarnings(preparedResult, diskWarnings); - return preparedResult; + auto result = PrepareHandout(std::move(handout), filePath); + PrependWarnings(result, warnings); + return result; } #if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS @@ -431,7 +429,9 @@ void Dream3dPreflightCache::SetForceFileMetadataFailure(bool forceFailure) void Dream3dPreflightCache::invalidate(const fs::path& filePath) { const std::lock_guard lock(m_Mutex); - m_Entries.erase(MakeKey(filePath)); + const auto key = MakeKey(filePath); + m_Entries.erase(CacheKey{key, false}); + m_Entries.erase(CacheKey{key, true}); } void Dream3dPreflightCache::clear() diff --git a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp index 968904cd4d..c11ce1c766 100644 --- a/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp +++ b/src/simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp @@ -14,6 +14,16 @@ namespace nx::core::DREAM3D { +/** + * @struct NeutralMetadataHandout + * @brief Carries isolated metadata and the version read from its source file. + */ +struct SIMPLNX_EXPORT NeutralMetadataHandout +{ + DataStructure dataStructure; + std::string fileVersion; +}; + /** * @class Dream3dPreflightCache * @brief Caches metadata-only DataStructures from DREAM3D files. @@ -22,7 +32,7 @@ namespace nx::core::DREAM3D * metadata traversal performs many small reads. Network storage can make that * traversal slow. A valid cache hit needs only filesystem metadata checks. * - * A canonical path identifies each entry. File size and modification time form + * A canonical path and mode identify each entry. File size and modification time form * its validation token. Recently modified files bypass the cache because a * network filesystem can round modification times. * @@ -72,7 +82,16 @@ class SIMPLNX_EXPORT Dream3dPreflightCache Result fetch(const std::filesystem::path& filePath); /** - * @brief Removes one file entry if present. + * @brief Gets isolated metadata without consulting process storage policy. + * @param filePath Identifies the source DREAM3D file. + * @return Metadata, file version, and disk warnings replayed once for this handout. + * + * Ordinary and neutral modes have separate masters and share the same capacity. + */ + Result fetchNeutralMetadata(const std::filesystem::path& filePath); + + /** + * @brief Removes all cache modes for one file if present. * @param filePath Identifies the entry through canonical path resolution. * * The next fetch reads the file even if its validation token did not change. @@ -116,6 +135,15 @@ class SIMPLNX_EXPORT Dream3dPreflightCache private: Dream3dPreflightCache() = default; + using CacheKey = std::pair; + + /** + * @brief Gets one handout through the shared cache mechanics. + * @param filePath Identifies the source. + * @param neutral Selects local resident metadata policy and disk-warning replay. + * @return Isolated handout and diagnostics. + */ + Result fetchMetadata(const std::filesystem::path& filePath, bool neutral); /** * @brief Replaces each array store with an independent handout store. @@ -134,7 +162,7 @@ class SIMPLNX_EXPORT Dream3dPreflightCache * @param filePath Identifies the source file for error context. * @return Prepared handout or contextual refresh error. */ - static Result PrepareHandout(DataStructure handout, const std::filesystem::path& filePath); + static Result PrepareHandout(NeutralMetadataHandout handout, const std::filesystem::path& filePath); /** * @brief Serves one valid cached entry and refreshes its recency. @@ -148,7 +176,7 @@ class SIMPLNX_EXPORT Dream3dPreflightCache * The method copies the master under m_Mutex, then isolates stores after it * releases the table lock. Both fast and post-read recheck paths use it. */ - Result> tryServeFromCache(const std::string& key, uint64 fileSize, const std::filesystem::file_time_type& mtime, const std::filesystem::path& filePath); + Result> tryServeFromCache(const CacheKey& key, uint64 fileSize, const std::filesystem::file_time_type& mtime, const std::filesystem::path& filePath); /** * @struct Entry @@ -159,7 +187,8 @@ class SIMPLNX_EXPORT Dream3dPreflightCache */ struct Entry { - DataStructure master; + NeutralMetadataHandout master; + std::vector diskWarnings; uint64 fileSize = 0; std::filesystem::file_time_type mtime; uint64 lastUsedTick = 0; @@ -174,7 +203,7 @@ class SIMPLNX_EXPORT Dream3dPreflightCache */ std::mutex m_ReadMutex; mutable std::mutex m_Mutex; - std::map m_Entries; + std::map m_Entries; uint64 m_Tick = 0; std::atomic m_Hits{0}; std::atomic m_Misses{0}; diff --git a/src/simplnx/Utilities/Parsing/HDF5/BoundedChunkRead.cpp b/src/simplnx/Utilities/Parsing/HDF5/BoundedChunkRead.cpp new file mode 100644 index 0000000000..881418a40f --- /dev/null +++ b/src/simplnx/Utilities/Parsing/HDF5/BoundedChunkRead.cpp @@ -0,0 +1,1071 @@ +#include "simplnx/Common/BoundedRead.hpp" +#include "simplnx/Common/Extent.hpp" +#include "simplnx/Common/Result.hpp" +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Types.hpp" +#include "simplnx/Utilities/Parsing/HDF5/H5Support.hpp" +#include "simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp" +#include "simplnx/Utilities/PositionalFileIO.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace nx::core::HDF5 +{ +namespace +{ +using BoundedRead::Diagnostic; +using BoundedRead::Status; + +/** @brief Labels the actual owned storage seen by the existing test observer. */ +enum class BoundedCarrier : uint64 +{ + Metadata, + Stored, + Nominal, + Decoder +}; + +/** @brief Carries allocation identity without copying a path or dataset. */ +struct BoundedIdentity +{ + const std::filesystem::path& File; + const std::string& Dataset; + +#if SIMPLNX_BUILD_TESTS + void observe(CodecIoEventForTesting event, uint64 bytes, BoundedCarrier carrier) const noexcept + { + ObserveCodecIoForTesting(File, Dataset, event, bytes, static_cast(carrier)); + } +#endif +}; + +/** @brief Stores each zlib allocation's admitted size in its own aligned header. */ +struct DecoderHeader +{ + alignas(std::max_align_t) uint64 Bytes = 0; +}; + +/** @brief Bounds zlib-owned storage; this state outlives inflateEnd and every zfree. */ +struct DecoderQuota +{ + const BoundedIdentity* Identity = nullptr; + uint64 Allowance = 0; + uint64 Live = 0; + bool QuotaFailed = false; + bool AllocationFailed = false; + + /** + * @brief Services a zlib request only after charging its payload and aligned header. + * @param opaque Borrows the live DecoderQuota. + * @param count Supplies zlib's element count. + * @param size Supplies bytes per element. + * @return Aligned payload, or null with a fixed quota/allocation failure flag. + */ + static voidpf Allocate(voidpf opaque, uInt count, uInt size) noexcept + { + auto& quota = *static_cast(opaque); + uint64 payload = 0; + uint64 bytes = 0; + if(!BoundedRead::Multiply(count, size, payload) || !BoundedRead::Add(payload, sizeof(DecoderHeader), bytes) || bytes > quota.Allowance - quota.Live || bytes > std::numeric_limits::max()) + { + quota.QuotaFailed = true; + return nullptr; + } +#if SIMPLNX_BUILD_TESTS + quota.Identity->observe(CodecIoEventForTesting::BoundedRequested, bytes, BoundedCarrier::Decoder); +#endif +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(quota.Identity->File, quota.Identity->Dataset, BoundedFaultPointForTesting::DecoderAllocation)) + { + quota.AllocationFailed = true; + return nullptr; + } +#endif + auto* headerPtr = static_cast(std::malloc(static_cast(bytes))); + if(headerPtr == nullptr) + { + quota.AllocationFailed = true; + return nullptr; + } + headerPtr->Bytes = bytes; + quota.Live += bytes; +#if SIMPLNX_BUILD_TESTS + quota.Identity->observe(CodecIoEventForTesting::BoundedAcquired, bytes, BoundedCarrier::Decoder); +#endif + return headerPtr + 1; + } + + /** + * @brief Frees one zlib payload before releasing its actual byte charge. + * @param opaque Borrows the quota that admitted this payload. + * @param address Supplies the payload returned by Allocate, or null. + */ + static void Free(voidpf opaque, voidpf address) noexcept + { + if(address == nullptr) + { + return; + } + auto& quota = *static_cast(opaque); + auto* headerPtr = static_cast(address) - 1; + const uint64 bytes = headerPtr->Bytes; + std::free(headerPtr); + quota.Live -= bytes; +#if SIMPLNX_BUILD_TESTS + quota.Identity->observe(CodecIoEventForTesting::BoundedReleased, bytes, BoundedCarrier::Decoder); +#endif + } +}; + +/** @brief Owns one exact requested byte allocation, without vector growth. */ +struct BoundedBytes +{ + explicit BoundedBytes(const BoundedIdentity& identityValue, BoundedCarrier carrierValue) noexcept + : Identity(identityValue) + , Carrier(carrierValue) + { + } + + BoundedBytes(const BoundedBytes&) = delete; + BoundedBytes& operator=(const BoundedBytes&) = delete; + BoundedBytes(BoundedBytes&&) = delete; + BoundedBytes& operator=(BoundedBytes&&) = delete; + + const BoundedIdentity& Identity; + BoundedCarrier Carrier; + std::byte* Data = nullptr; + uint64 Size = 0; + + ~BoundedBytes() noexcept + { + if(Data != nullptr) + { + ::operator delete(Data); +#if SIMPLNX_BUILD_TESTS + Identity.observe(CodecIoEventForTesting::BoundedReleased, Size, Carrier); +#endif + } + } + + /** + * @brief Requests the exact byte count after the caller has admitted it. + * @param bytes Supplies the checked, addressable allocation count. + * @return False on allocation failure; true for zero bytes without allocation. + * @pre This carrier has no previous allocation. + */ + bool acquire(uint64 bytes) noexcept + { + if(bytes == 0) + { + return true; + } +#if SIMPLNX_BUILD_TESTS + Identity.observe(CodecIoEventForTesting::BoundedRequested, bytes, Carrier); +#endif +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(Identity.File, Identity.Dataset, Carrier == BoundedCarrier::Stored ? BoundedFaultPointForTesting::StoredAllocation : BoundedFaultPointForTesting::NominalAllocation)) + { + return false; + } +#endif + Data = static_cast(::operator new(static_cast(bytes), std::nothrow)); + if(Data == nullptr) + { + return false; + } + Size = bytes; +#if SIMPLNX_BUILD_TESTS + Identity.observe(CodecIoEventForTesting::BoundedAcquired, Size, Carrier); +#endif + return true; + } +}; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(!std::is_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +/** @brief Holds constant-rank metadata for two physical-record passes and scatter. */ +struct BoundedChunkReadMetadata +{ + explicit BoundedChunkReadMetadata(const BoundedIdentity& identity) + : Stored{identity, BoundedCarrier::Stored} + , Nominal{identity, BoundedCarrier::Nominal} + { + } + + ~BoundedChunkReadMetadata() = default; + BoundedChunkReadMetadata(const BoundedChunkReadMetadata&) = delete; + BoundedChunkReadMetadata(BoundedChunkReadMetadata&&) = delete; + BoundedChunkReadMetadata& operator=(const BoundedChunkReadMetadata&) = delete; + BoundedChunkReadMetadata& operator=(BoundedChunkReadMetadata&&) = delete; + + BoundedBytes Stored; + BoundedBytes Nominal; + std::array Shape{}; + std::array Chunks{}; + std::array Origin{}; + std::array First{}; + std::array Last{}; + std::array Counts{}; + std::array NominalStrides{}; + std::array OutputStrides{}; + std::array Low{}; + std::array High{}; + std::array Cursor{}; + std::array Selected{}; + ParallelChunkCodec::ChunkInfo Record; + z_stream Stream{}; + DecoderQuota Quota; + std::array Fill{}; + uint64 NominalBytes = 0; + uint64 MaximumStored = 0; + usize Rank = 0; + bool HasDeflate = false; + bool NeedsDecoder = false; + bool HasSparse = false; +}; +static_assert(H5S_MAX_RANK <= BoundedRead::k_MaxRank); + +/** @brief Bounds a cleanup guard's two-reference capture and active flag on the compiling ABI. */ +struct BoundedCleanupControl +{ + std::array Captures{}; + bool Active = false; +}; + +/** @brief Inventories the fixed native-query carriers that coexist with chunk metadata. */ +struct BoundedNativeQueryControls +{ + std::array Identifiers{}; + std::array Statuses{}; + std::array FilterValuesAndFlags{}; + std::array CountsAndSize{}; + H5D_layout_t Layout = H5D_LAYOUT_ERROR; + H5D_fill_value_t Fill = H5D_FILL_VALUE_ERROR; + haddr_t Address = HADDR_UNDEF; + hsize_t StoredSize = 0; + unsigned Mask = 0; +}; + +/** @brief Keeps all scatter loop state in one admitted fixed-size control. */ +struct BoundedScatterControl +{ + uint64 RunValues = 1; + uint64 JoinedValues = 0; + uint64 SourceIndex = 0; + uint64 OutputIndex = 0; + usize FirstRunDimension = 0; + usize Dimension = 0; + usize Reverse = 0; +}; + +// Scatter borrows two references and two byte pointers, plus scalar width and sparse mode. +// Multiply borrows one result reference and accepts two scalar operands. +constexpr uint64 k_BoundedScatterArgumentBytes = (4 * sizeof(void*)) + sizeof(usize) + sizeof(bool) + sizeof(void*) + (2 * sizeof(uint64)); +constexpr uint64 k_BoundedScatterBytes = sizeof(BoundedScatterControl) + k_BoundedScatterArgumentBytes; + +// ScopeGuard has one captured callable and an active flag. The two additional +// capture slots per guard cover constructor arguments and source temporaries. +// Four guards cover metadata release, properties, decoder and the POSIX handle. +constexpr uint64 k_BoundedCleanupBytes = 4 * (sizeof(BoundedCleanupControl) + 2 * sizeof(std::array)); +constexpr uint64 k_BoundedReadClosureBytes = 5 * sizeof(void*); +constexpr uint64 k_BoundedControlBytes = sizeof(BoundedIdentity) + sizeof(hid_t) + sizeof(bool) + sizeof(detail::FileHandle) + sizeof(BoundedNativeQueryControls) + + (2 * sizeof(std::lock_guard)) + k_BoundedCleanupBytes + k_BoundedReadClosureBytes + (2 * sizeof(nonstd::span)) + + sizeof(nonstd::span) + (2 * sizeof(Status)) + k_BoundedScatterBytes +#if SIMPLNX_BUILD_TESTS + + sizeof(BoundedReadTaskScopeForTesting) + sizeof(MetadataIdentityForTesting) +#endif + ; +constexpr uint64 k_BoundedFixedBytes = sizeof(BoundedChunkReadMetadata) + k_BoundedControlBytes; + +/** + * @brief Reads one record without treating a failed query as sparse storage. + * @param dataset Supplies the open dataset identifier. + * @param origin Supplies an aligned full-rank chunk origin. + * @param record Receives checked allocation, length and filter state. + * @param diagnostic Receives fixed query-failure context. + * @param identity Borrows the scoped test identity. + * @return Complete or a metadata failure. + * @pre The caller holds the HDF5 API lock. + */ +Status queryRecord(hid_t dataset, const hsize_t* origin, ParallelChunkCodec::ChunkInfo& record, Diagnostic& diagnostic, const BoundedIdentity& identity) +{ + haddr_t address = HADDR_UNDEF; + hsize_t bytes = 0; + unsigned mask = 0; + if(H5Dget_chunk_info_by_coord(dataset, origin, &mask, &address, &bytes) < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk metadata query failed."); + } +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(identity.File, identity.Dataset, BoundedFaultPointForTesting::ContradictoryRecord)) + { + address = HADDR_UNDEF; + bytes = 1; + } +#else + static_cast(identity); +#endif + if((address == HADDR_UNDEF) != (bytes == 0)) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk metadata has contradictory address and length."); + } + record = {address != HADDR_UNDEF, mask, static_cast(address), static_cast(bytes)}; + return Status::Complete; +} + +/** + * @brief Advances to the next physical chunk that contains a selected coordinate. + * @param metadata Owns the fixed-rank cursor and checked dimensions. + * @param extent Supplies the original strided selection. + * @return False after the last selected chunk; the cursor then resets. + */ +bool nextChunk(BoundedChunkReadMetadata& metadata, const Extent& extent) noexcept +{ + for(usize reverse = metadata.Rank; reverse != 0; --reverse) + { + const usize dimension = reverse - 1; + const uint64 end = metadata.Origin[dimension] + std::min(metadata.Chunks[dimension], metadata.Shape[dimension] - metadata.Origin[dimension]) - 1; + const uint64 nextSelected = ((end - extent.min[dimension]) / extent.stride[dimension]) + 1; + if(nextSelected < metadata.Counts[dimension]) + { + const uint64 coordinate = extent.min[dimension] + (nextSelected * extent.stride[dimension]); + metadata.Origin[dimension] = (coordinate / metadata.Chunks[dimension]) * metadata.Chunks[dimension]; + return true; + } + metadata.Origin[dimension] = metadata.First[dimension]; + } + return false; +} + +/** + * @brief Intersects one physical chunk with the strided selection. + * @param metadata Receives selected-index bounds and the scatter cursor. + * @param extent Supplies tuple and component coordinates. + * @return True when the chunk contains a selected scalar. + */ +bool intersect(BoundedChunkReadMetadata& metadata, const Extent& extent) noexcept +{ + for(usize dimension = 0; dimension < metadata.Rank; ++dimension) + { + const uint64 chunkEnd = metadata.Origin[dimension] + std::min(metadata.Chunks[dimension], metadata.Shape[dimension] - metadata.Origin[dimension]) - 1; + const uint64 difference = metadata.Origin[dimension] > extent.min[dimension] ? metadata.Origin[dimension] - extent.min[dimension] : 0; + metadata.Low[dimension] = difference / extent.stride[dimension] + (difference % extent.stride[dimension] != 0 ? 1 : 0); + if(chunkEnd < extent.min[dimension]) + { + return false; + } + metadata.High[dimension] = (std::min(chunkEnd, extent.max[dimension]) - extent.min[dimension]) / extent.stride[dimension]; + if(metadata.Low[dimension] > metadata.High[dimension]) + { + return false; + } + metadata.Cursor[dimension] = metadata.Low[dimension]; + } + return true; +} + +/** + * @brief Validates the bounded raw representation. + * @param metadata Supplies checked layout and nominal bytes. + * @param record Supplies current allocation, filter and stored-byte state. + * @param diagnostic Receives fixed validation context. + * @return Complete or an invalid metadata/length result. + * @note The ordinary decoder retains its separate length and trailing-byte policy. + */ +Status validateRecord(const BoundedChunkReadMetadata& metadata, const ParallelChunkCodec::ChunkInfo& record, Diagnostic& diagnostic) +{ + if(!record.allocated) + { + return Status::Complete; + } + if((record.filterMask & ~(metadata.HasDeflate ? 1U : 0U)) != 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk has a filter mask outside its declared pipeline."); + } + const bool compressed = metadata.HasDeflate && record.filterMask == 0; + if(!compressed && record.storedSize != metadata.NominalBytes) + { + return diagnostic.counts(BoundedRead::MetadataFailure, "Bounded raw chunk byte length does not match nominal storage.", metadata.NominalBytes, record.storedSize); + } + if(record.storedSize > std::numeric_limits::max() || record.storedSize > static_cast(std::numeric_limits::max()) || + (compressed && (record.storedSize > std::numeric_limits::max() || metadata.NominalBytes > std::numeric_limits::max()))) + { + return diagnostic.set(BoundedRead::InvalidExtent, "Bounded chunk exceeds the supported raw or zlib length."); + } + return Status::Complete; +} + +/** + * @brief Copies selected bytes with runs that are contiguous in both source and destination. + * @param metadata Supplies checked nominal strides, intersection bounds and scalar fill bytes. + * @param extent Supplies source coordinates and selection strides. + * @param elementSize Supplies the validated scalar width. + * @param source Borrows nominal bytes when sparse is false. + * @param sparse Selects the checked scalar fill value instead of source. + * @param destination Receives selected bytes at their row-major output positions. + */ +void scatter(BoundedChunkReadMetadata& metadata, const Extent& extent, usize elementSize, const std::byte* source, bool sparse, std::byte* destination) noexcept +{ + BoundedScatterControl control; + static_assert(sizeof(control) + k_BoundedScatterArgumentBytes <= k_BoundedScatterBytes); + control.FirstRunDimension = metadata.Rank; + if(!sparse && extent.stride[metadata.Rank - 1] == 1) + { + control.FirstRunDimension = metadata.Rank - 1; + control.RunValues = metadata.High[control.FirstRunDimension] - metadata.Low[control.FirstRunDimension] + 1; + while(control.FirstRunDimension != 0) + { + control.Dimension = control.FirstRunDimension - 1; + // Nominal strides retain edge padding. Output strides retain gaps from split components and partial rows. + if(extent.stride[control.Dimension] != 1 || metadata.NominalStrides[control.Dimension] != control.RunValues || metadata.OutputStrides[control.Dimension] != control.RunValues || + !BoundedRead::Multiply(control.RunValues, metadata.High[control.Dimension] - metadata.Low[control.Dimension] + 1, control.JoinedValues)) + { + break; + } + control.RunValues = control.JoinedValues; + --control.FirstRunDimension; + } + } + // Each run stays within checked nominal/output products, so its scalar-to-byte conversion fits usize. + // Sparse fills remain scalar copies from the fixed primitive fill buffer. + for(;;) + { + control.SourceIndex = 0; + control.OutputIndex = 0; + for(control.Dimension = 0; control.Dimension < metadata.Rank; ++control.Dimension) + { + control.SourceIndex += + (extent.min[control.Dimension] + metadata.Cursor[control.Dimension] * extent.stride[control.Dimension] - metadata.Origin[control.Dimension]) * metadata.NominalStrides[control.Dimension]; + control.OutputIndex += metadata.Cursor[control.Dimension] * metadata.OutputStrides[control.Dimension]; + } + std::memcpy(destination + (static_cast(control.OutputIndex) * elementSize), sparse ? metadata.Fill.data() : source + (static_cast(control.SourceIndex) * elementSize), + static_cast(control.RunValues) * elementSize); + control.Reverse = control.FirstRunDimension; + while(control.Reverse != 0) + { + control.Dimension = control.Reverse - 1; + if(metadata.Cursor[control.Dimension] < metadata.High[control.Dimension]) + { + ++metadata.Cursor[control.Dimension]; + break; + } + metadata.Cursor[control.Dimension] = metadata.Low[control.Dimension]; + --control.Reverse; + } + if(control.Reverse == 0) + { + return; + } + } +} +} // namespace + +#if SIMPLNX_BUILD_TESTS +uint64 ParallelChunkCodec::boundedMetadataCapacityBytesForTesting() const noexcept +{ + return ((m_FilePath.native().capacity() + 1) * sizeof(std::filesystem::path::value_type)) + m_DatasetPath.capacity() + 1 + + ((m_TupleShape.capacity() + m_ChunkShape.capacity() + m_ComponentShape.capacity()) * sizeof(uint64)); +} +#endif + +Result ParallelChunkCodec::getChunkInfoChecked(nonstd::span fullRankChunkOffset) const +{ + bool representationSupported = BoundedRead::DiagnosticRepresentationSupported(); +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::CheckedMetadataUnsupportedBuild)) + { + representationSupported = false; + } +#endif + if(!representationSupported) + { + // Mandatory empty error/warning controls only. No value, element or text is constructed. + return {{nonstd::make_unexpected(ErrorCollection{})}}; + } +#if SIMPLNX_BUILD_TESTS + const MetadataIdentityForTesting metadataIdentity(m_FilePath, m_DatasetPath); +#endif + Diagnostic diagnostic; + ChunkInfo info; + Status status = Status::Complete; + std::array offset{}; + std::array dimensions{}; + std::array chunks{}; + if(fullRankChunkOffset.empty() || fullRankChunkOffset.size() > offset.size()) + { + status = diagnostic.set(BoundedRead::InvalidExtent, "Checked chunk origin has an invalid rank."); + } + else + { + const std::lock_guard hdf5Lock(Support::ApiLock()); + const hid_t space = SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Dget_space(m_DatasetId)); + const hid_t properties = SIMPLNX_OBSERVE_METADATA_CALL(MetadataLayoutCall, H5Dget_create_plist(m_DatasetId)); + auto closeMetadata = MakeScopeGuard([space, properties]() noexcept { + if(space >= 0) + { + H5Sclose(space); + } + if(properties >= 0) + { + H5Pclose(properties); + } + }); + if(space < 0 || properties < 0) + { + status = diagnostic.set(BoundedRead::MetadataFailure, "Checked chunk query cannot obtain dataset metadata."); + } + else + { + const int rank = SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Sget_simple_extent_ndims(space)); + if(rank < 0) + { + status = diagnostic.set(BoundedRead::MetadataFailure, "Checked chunk query cannot obtain the dataset rank."); + } + else if(rank != static_cast(fullRankChunkOffset.size())) + { + status = diagnostic.set(BoundedRead::InvalidExtent, "Checked chunk origin does not match the dataset rank."); + } + else if(SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Sget_simple_extent_dims(space, dimensions.data(), nullptr)) != rank || + SIMPLNX_OBSERVE_METADATA_CALL(MetadataLayoutCall, H5Pget_chunk(properties, rank, chunks.data())) != rank) + { + status = diagnostic.set(BoundedRead::MetadataFailure, "Checked chunk query cannot obtain dimension and chunk sizes."); + } + else + { + for(usize dimension = 0; dimension < fullRankChunkOffset.size(); ++dimension) + { + if(chunks[dimension] == 0 || fullRankChunkOffset[dimension] >= dimensions[dimension] || fullRankChunkOffset[dimension] % chunks[dimension] != 0) + { + status = diagnostic.set(BoundedRead::InvalidExtent, "Checked chunk origin is outside or not aligned to the physical chunk grid."); + break; + } + offset[dimension] = fullRankChunkOffset[dimension]; + } + if(status == Status::Complete) + { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::ChunkRecordQuery, 1, 0); +#endif + status = queryRecord(m_DatasetId, offset.data(), info, diagnostic, BoundedIdentity{.File = m_FilePath, .Dataset = m_DatasetPath}); + } + } + } + } + Result result{info}; + if(status != Status::Complete) + { + diagnostic.context(m_FilePath, m_DatasetPath); + auto failure = diagnostic.finish(status); + result.m_Expected = nonstd::make_unexpected(std::move(failure.errors())); + } + return result; +} + +Result ParallelChunkCodec::readExtentIntoBufferBounded(const Extent& extent, nonstd::span destination, uint64 scratchBudgetBytes) const +{ + if(scratchBudgetBytes < BoundedRead::k_DiagnosticBytes || !BoundedRead::DiagnosticRepresentationSupported()) + { + return {false}; + } + Diagnostic diagnostic; + const auto status = readExtentIntoBufferBoundedState(extent, destination, scratchBudgetBytes - BoundedRead::k_DiagnosticBytes, diagnostic); + if(status != Status::Complete) + { + diagnostic.context(m_FilePath, m_DatasetPath); + } + return diagnostic.finish(status); +} + +BoundedRead::Status ParallelChunkCodec::readExtentIntoBufferBoundedState(const Extent& extent, nonstd::span destination, uint64 payloadBudgetBytes, + BoundedRead::Diagnostic& diagnostic) const +{ + if(payloadBudgetBytes < k_BoundedFixedBytes) + { + return diagnostic.counts(BoundedRead::Insufficient, "Bounded chunk read cannot admit fixed metadata and controls.", k_BoundedFixedBytes, payloadBudgetBytes); + } + const BoundedIdentity identity{.File = m_FilePath, .Dataset = m_DatasetPath}; +#if SIMPLNX_BUILD_TESTS + const BoundedReadTaskScopeForTesting taskObservation; + auto metadataRelease = MakeScopeGuard([&identity]() noexcept { identity.observe(CodecIoEventForTesting::BoundedReleased, sizeof(BoundedChunkReadMetadata), BoundedCarrier::Metadata); }); + identity.observe(CodecIoEventForTesting::BoundedRequested, sizeof(BoundedChunkReadMetadata), BoundedCarrier::Metadata); +#endif +#if SIMPLNX_BUILD_TESTS + static_assert(sizeof(metadataRelease) <= sizeof(BoundedCleanupControl)); + identity.observe(CodecIoEventForTesting::BoundedAdmissionCharge, k_BoundedControlBytes, BoundedCarrier::Metadata); +#endif + BoundedChunkReadMetadata metadata(identity); +#if SIMPLNX_BUILD_TESTS + identity.observe(CodecIoEventForTesting::BoundedAcquired, sizeof(metadata), BoundedCarrier::Metadata); + const MetadataIdentityForTesting metadataIdentity(m_FilePath, m_DatasetPath); +#endif + const uint64 available = payloadBudgetBytes - k_BoundedFixedBytes; + metadata.Quota.Identity = &identity; + hid_t properties = H5I_INVALID_HID; + auto closeProperties = MakeScopeGuard([&properties]() noexcept { + if(properties >= 0) + { + const std::lock_guard lock(Support::ApiLock()); + H5Pclose(properties); + } + }); + static_assert(sizeof(closeProperties) <= sizeof(BoundedCleanupControl)); + { + const std::lock_guard lock(Support::ApiLock()); + if(m_BoundedTypeQueryFailed) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded native type capture failed."); + } + if(m_BoundedNativeType < 0) + { + return diagnostic.set(BoundedRead::Unsupported, "Bounded chunk reads require an exact native numeric memory type."); + } + usize nativeSize = H5Tget_size(m_BoundedNativeType); +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::NativeSizeQuery)) + { + nativeSize = 0; + } +#endif + if(nativeSize == 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded native scalar-size query failed."); + } + if(m_ElementSize == 0 || nativeSize != m_ElementSize) + { + return diagnostic.counts(BoundedRead::InvalidExtent, "Bounded native type size does not match scalar bytes.", nativeSize, m_ElementSize); + } + const hid_t fileType = SIMPLNX_OBSERVE_METADATA_CALL(MetadataTypeCall, H5Dget_type(m_DatasetId)); + if(fileType < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk type query failed."); + } + const htri_t exact = H5Tequal(fileType, m_BoundedNativeType); + H5Tclose(fileType); + if(exact < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk type comparison failed."); + } + if(exact == 0) + { + return diagnostic.set(BoundedRead::Unsupported, "Bounded chunk read requires datatype conversion."); + } + properties = SIMPLNX_OBSERVE_METADATA_CALL(MetadataLayoutCall, H5Dget_create_plist(m_DatasetId)); + if(properties < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk layout query failed."); + } + const H5D_layout_t layout = SIMPLNX_OBSERVE_METADATA_CALL(MetadataLayoutCall, H5Pget_layout(properties)); + const int external = H5Pget_external_count(properties); + if(layout == H5D_LAYOUT_ERROR || external < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk layout or external-storage query failed."); + } + if(layout != H5D_CHUNKED || external != 0) + { + return diagnostic.set(BoundedRead::Unsupported, "Bounded codec requires internal chunked storage."); + } + const int filters = SIMPLNX_OBSERVE_METADATA_CALL(MetadataFilterCall, H5Pget_nfilters(properties)); + if(filters < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk filter count query failed."); + } + if(filters > 1) + { + return diagnostic.counts(BoundedRead::UnsupportedFilter, "Bounded chunk pipeline exceeds the supported filter count.", static_cast(filters), 1); + } + if(filters == 1) + { + unsigned flags = 0; + unsigned configuration = 0; + std::array values{}; + size_t count = values.size(); + const H5Z_filter_t filter = SIMPLNX_OBSERVE_METADATA_CALL(MetadataFilterCall, H5Pget_filter2(properties, 0, &flags, &count, values.data(), 0, nullptr, &configuration)); + if(filter == H5Z_FILTER_ERROR) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk filter query failed."); + } + if(filter != H5Z_FILTER_DEFLATE || count != 1 || values[0] > 9) + { + diagnostic.set(BoundedRead::UnsupportedFilter, "Bounded chunk pipeline is not a supported single deflate filter; filter="); + diagnostic.number(static_cast(filter)); + diagnostic.append(", parameter count="); + diagnostic.number(count); + diagnostic.append(", first parameter="); + diagnostic.number(values[0]); + return Status::Unavailable; + } + metadata.HasDeflate = true; + } + const hid_t space = SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Dget_space(m_DatasetId)); + if(space < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk dataspace query failed."); + } + const int rank = H5Sget_simple_extent_ndims(space); + const int dimensions = rank > 0 && rank <= H5S_MAX_RANK ? H5Sget_simple_extent_dims(space, metadata.Shape.data(), nullptr) : -1; + H5Sclose(space); + if(rank < 0 || (rank > 0 && rank <= H5S_MAX_RANK && dimensions < 0)) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded chunk rank or dimensions query failed."); + } + if(rank == 0 || rank > H5S_MAX_RANK) + { + return diagnostic.set(BoundedRead::Unsupported, "Bounded chunk read does not support this rank."); + } + metadata.Rank = static_cast(rank); + if(H5Pget_chunk(properties, rank, metadata.Chunks.data()) != rank) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded physical chunk shape query failed."); + } + } + uint64 selectedValues = 0; + if(destination.size() % m_ElementSize != 0) + { + return diagnostic.set(BoundedRead::InvalidExtent, "Bounded byte destination is not a scalar multiple."); + } + const auto extentStatus = + BoundedRead::ValidateExtent(extent, nonstd::span(metadata.Shape.data(), metadata.Rank), 1, destination.size() / m_ElementSize, selectedValues, diagnostic, m_ElementSize); + if(extentStatus != Status::Complete) + { + return extentStatus; + } + uint64 nominalValues = 1; + uint64 outputStride = 1; + for(usize reverse = metadata.Rank; reverse != 0; --reverse) + { + const usize dimension = reverse - 1; + if(metadata.Chunks[dimension] == 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded physical chunk shape has a zero dimension."); + } + metadata.NominalStrides[dimension] = nominalValues; + metadata.OutputStrides[dimension] = outputStride; + metadata.Counts[dimension] = (extent.max[dimension] - extent.min[dimension]) / extent.stride[dimension] + 1; + if(!BoundedRead::Multiply(nominalValues, metadata.Chunks[dimension], nominalValues) || !BoundedRead::Multiply(outputStride, metadata.Counts[dimension], outputStride)) + { + return diagnostic.set(BoundedRead::InvalidExtent, "Bounded chunk strides overflow uint64."); + } + metadata.First[dimension] = (extent.min[dimension] / metadata.Chunks[dimension]) * metadata.Chunks[dimension]; + metadata.Last[dimension] = (extent.max[dimension] / metadata.Chunks[dimension]) * metadata.Chunks[dimension]; + metadata.Origin[dimension] = metadata.First[dimension]; + } + if(!BoundedRead::Multiply(nominalValues, m_ElementSize, metadata.NominalBytes) || metadata.NominalBytes > std::numeric_limits::max()) + { + return diagnostic.set(BoundedRead::InvalidExtent, "Bounded nominal chunk bytes overflow addressable storage."); + } + // Pass one admits every touched record under the existing serial-alias precondition. + do + { + if(!intersect(metadata, extent)) + { + continue; + } + { + const std::lock_guard lock(Support::ApiLock()); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::ChunkRecordQuery, 1, 0); +#endif +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::MetadataQuery)) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Injected bounded metadata query failure."); + } +#endif + const auto status = queryRecord(m_DatasetId, metadata.Origin.data(), metadata.Record, diagnostic, identity); + if(status != Status::Complete) + { + return status; + } + } + const auto recordStatus = validateRecord(metadata, metadata.Record, diagnostic); + if(recordStatus != Status::Complete) + { + return recordStatus; + } + if(metadata.Record.allocated) + { + metadata.MaximumStored = std::max(metadata.MaximumStored, metadata.Record.storedSize); + metadata.NeedsDecoder = metadata.NeedsDecoder || (metadata.HasDeflate && metadata.Record.filterMask == 0); + } + else + { + metadata.HasSparse = true; + } + } while(nextChunk(metadata, extent)); + if(metadata.HasSparse) + { + const std::lock_guard lock(Support::ApiLock()); + H5D_fill_value_t fillStatus = H5D_FILL_VALUE_ERROR; + if(SIMPLNX_OBSERVE_METADATA_CALL(MetadataFillCall, H5Pfill_value_defined(properties, &fillStatus)) < 0 || fillStatus == H5D_FILL_VALUE_ERROR) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded fill-value status query failed."); + } + if(fillStatus == H5D_FILL_VALUE_UNDEFINED) + { + return diagnostic.set(BoundedRead::UndefinedFill, "Bounded sparse chunk has an undefined fill value."); + } + if(SIMPLNX_OBSERVE_METADATA_CALL(MetadataFillCall, H5Pget_fill_value(properties, m_BoundedNativeType, metadata.Fill.data())) < 0) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Bounded fill-value conversion failed."); + } + } + uint64 carrierBytes = 0; + const uint64 nominalAllocation = metadata.NeedsDecoder ? metadata.NominalBytes : 0; + if(!BoundedRead::Add(metadata.MaximumStored, nominalAllocation, carrierBytes)) + { + return diagnostic.set(BoundedRead::InvalidExtent, "Bounded raw and nominal carrier byte sum overflows uint64."); + } + if(carrierBytes > available) + { + return diagnostic.counts(BoundedRead::Insufficient, "Bounded raw and nominal carrier bytes exceed the allowance.", carrierBytes, available); + } + metadata.Quota.Allowance = available - carrierBytes; + metadata.Stream.zalloc = &DecoderQuota::Allocate; + metadata.Stream.zfree = &DecoderQuota::Free; + metadata.Stream.opaque = &metadata.Quota; + bool initialized = false; + auto finishDecoder = MakeScopeGuard([&metadata, &initialized]() noexcept { + if(initialized) + { + inflateEnd(&metadata.Stream); + } + }); + static_assert(sizeof(finishDecoder) <= sizeof(BoundedCleanupControl)); + if(metadata.NeedsDecoder) + { + const int status = inflateInit(&metadata.Stream); + initialized = status == Z_OK; + if(!initialized) + { + return diagnostic.set(metadata.Quota.QuotaFailed ? BoundedRead::Insufficient : BoundedRead::AllocationFailure, "Bounded zlib initialization cannot acquire admitted state."); + } + constexpr uint64 windowBytes = 32768 + sizeof(DecoderHeader); + if(windowBytes > metadata.Quota.Allowance - metadata.Quota.Live) + { + return diagnostic.counts(BoundedRead::Insufficient, "Bounded zlib state and maximum-window bytes exceed the allowance.", metadata.Quota.Live + windowBytes, metadata.Quota.Allowance); + } + } + auto& stored = metadata.Stored; + auto& nominal = metadata.Nominal; + if(!stored.acquire(metadata.MaximumStored) || !nominal.acquire(nominalAllocation)) + { + return diagnostic.set(BoundedRead::AllocationFailure, "Bounded chunk carrier allocation failed within its allowance."); + } +#ifndef _WIN32 + // Borrow the POSIX native string; opening this independent descriptor allocates no C++ path carrier. + const auto readHandle = metadata.MaximumStored == 0 ? detail::invalidFileHandle() : detail::openFileForRead(m_FilePath.native()); + auto closeReadHandle = MakeScopeGuard([readHandle]() noexcept { + if(detail::isValidFileHandle(readHandle)) + { + detail::closeFileHandle(readHandle); + } + }); + static_assert(sizeof(closeReadHandle) <= sizeof(BoundedCleanupControl)); + if(metadata.MaximumStored != 0 && !detail::isValidFileHandle(readHandle)) + { + return diagnostic.set(BoundedRead::ReadFailure, "Bounded positional source could not open."); + } +#endif + // Pass two checks current records against admitted limits. This is not a cross-alias snapshot. + do + { + if(!intersect(metadata, extent)) + { + continue; + } + { + const std::lock_guard lock(Support::ApiLock()); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::ChunkRecordQuery, 1, 0); +#endif +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::MetadataQuery)) + { + return diagnostic.set(BoundedRead::MetadataFailure, "Injected bounded metadata query failure."); + } +#endif + const auto status = queryRecord(m_DatasetId, metadata.Origin.data(), metadata.Record, diagnostic, identity); + if(status != Status::Complete) + { + return status; + } +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::SecondPassUnpreparedDecoder)) + { + metadata.Record.filterMask = 0; + } + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::SecondPassLargerRecord)) + { + metadata.Record.storedSize = metadata.MaximumStored + 1; + } +#endif + if(metadata.Record.storedSize > metadata.MaximumStored || (metadata.Record.allocated && metadata.HasDeflate && metadata.Record.filterMask == 0 && !initialized) || + (!metadata.Record.allocated && !metadata.HasSparse)) + { + diagnostic.counts(BoundedRead::MetadataChanged, "Bounded second-pass record requires unadmitted stored bytes or decoder/fill state.", metadata.Record.storedSize, metadata.MaximumStored); + diagnostic.append(", filter mask="); + diagnostic.number(metadata.Record.filterMask); + diagnostic.append(", decoder prepared="); + diagnostic.number(initialized ? 1 : 0); + diagnostic.append(", fill prepared="); + diagnostic.number(metadata.HasSparse ? 1 : 0); + return Status::Failed; + } + const auto recordStatus = validateRecord(metadata, metadata.Record, diagnostic); + if(recordStatus != Status::Complete) + { + return recordStatus; + } +#ifdef _WIN32 + if(metadata.Record.allocated) + { + unsigned ignoredMask = 0; +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadAttempt, metadata.Record.storedSize, 0); +#endif + // H5Dread_chunk has no capacity argument. Query, length check, and read share this lock. + bool injectedFailure = false; +#if SIMPLNX_BUILD_TESTS + injectedFailure = InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::RawRead); +#endif + if(injectedFailure || H5Dread_chunk(m_DatasetId, H5P_DEFAULT, metadata.Origin.data(), &ignoredMask, stored.Data) < 0) + { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadFailed, metadata.Record.storedSize, 0); +#endif + return diagnostic.set(BoundedRead::ReadFailure, "Bounded raw HDF5 chunk read failed."); + } +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredRead, metadata.Record.storedSize, 0); +#endif + } +#endif + } + if(!metadata.Record.allocated) + { + scatter(metadata, extent, m_ElementSize, nullptr, true, destination.data()); + continue; + } +#ifndef _WIN32 + auto readStored = [&]() { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadAttempt, metadata.Record.storedSize, 0); +#endif + bool injectedFailure = false; +#if SIMPLNX_BUILD_TESTS + injectedFailure = InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::RawRead); +#endif + const auto got = injectedFailure ? std::ptrdiff_t{-1} : detail::positionalRead(readHandle, stored.Data, static_cast(metadata.Record.storedSize), metadata.Record.storedAddress); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, + got == static_cast(metadata.Record.storedSize) ? CodecIoEventForTesting::StoredRead : + got < 0 ? CodecIoEventForTesting::StoredReadFailed : + CodecIoEventForTesting::StoredReadShort, + got >= 0 ? static_cast(got) : metadata.Record.storedSize, metadata.Record.storedSize); +#endif + return got; + }; + static_assert(sizeof(readStored) <= k_BoundedReadClosureBytes); + auto got = readStored(); + if(got != static_cast(metadata.Record.storedSize)) + { + std::lock_guard recoveryLock(m_PositionalReadRecoveryMutex); + got = readStored(); + if(got != static_cast(metadata.Record.storedSize)) + { + { + const std::lock_guard lock(Support::ApiLock()); + if(H5Fflush(m_DatasetId, H5F_SCOPE_LOCAL) < 0) + { + return diagnostic.set(BoundedRead::ReadFailure, "Bounded short-read recovery flush failed."); + } + } + got = readStored(); + } + } + if(got != static_cast(metadata.Record.storedSize)) + { + return diagnostic.set(BoundedRead::ReadFailure, "Bounded positional chunk read was short or failed."); + } +#endif + const std::byte* decodedPtr = stored.Data; + if(metadata.HasDeflate && metadata.Record.filterMask == 0) + { + if(inflateReset(&metadata.Stream) != Z_OK) + { + return diagnostic.set(BoundedRead::DecodeFailure, "Bounded decoder reset failed."); + } + metadata.Stream.next_in = reinterpret_cast(stored.Data); + metadata.Stream.avail_in = static_cast(metadata.Record.storedSize); + metadata.Stream.next_out = reinterpret_cast(nominal.Data); + metadata.Stream.avail_out = static_cast(metadata.NominalBytes); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::InflateAttempt, metadata.Record.storedSize, metadata.NominalBytes); +#endif + int status = Z_OK; +#if SIMPLNX_BUILD_TESTS + if(InjectCodecFaultForTesting(m_FilePath, m_DatasetPath, BoundedFaultPointForTesting::DecoderQuotaAfterPayload)) + { + // Withhold the last checksum byte to exercise zlib's real window request. + // This deterministic fault changes only the admitted test allocator limit. + metadata.Quota.Allowance = metadata.Quota.Live; + --metadata.Stream.avail_in; + status = inflate(&metadata.Stream, Z_NO_FLUSH); + } + else +#endif + { + status = inflate(&metadata.Stream, Z_FINISH); + } + if(status != Z_STREAM_END || metadata.Stream.total_out != metadata.NominalBytes || metadata.Stream.avail_in != 0) + { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::InflateFailed, metadata.Record.storedSize, metadata.Stream.total_out); +#endif + return diagnostic.set(metadata.Quota.QuotaFailed ? BoundedRead::DecoderQuotaFailure : + metadata.Quota.AllocationFailed ? BoundedRead::AllocationFailure : + BoundedRead::DecodeFailure, + "Bounded decoder did not consume one complete nominal stream."); + } +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::Inflated, metadata.Record.storedSize, metadata.Stream.total_out); +#endif + decodedPtr = nominal.Data; + } + else if(metadata.HasDeflate) + { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::SkippedDeflate, metadata.Record.storedSize, metadata.NominalBytes); +#endif + } + scatter(metadata, extent, m_ElementSize, decodedPtr, false, destination.data()); + } while(nextChunk(metadata, extent)); + return Status::Complete; +} +} // namespace nx::core::HDF5 diff --git a/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.cpp b/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.cpp index b25b0ec17e..fdd864c360 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.cpp +++ b/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.cpp @@ -1,6 +1,10 @@ #include "simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp" +#include "simplnx/Common/Aliases.hpp" +#include "simplnx/Common/Types.hpp" +#include #include +#include namespace { @@ -18,7 +22,7 @@ namespace * A row includes the full extent of each dimension from firstRowDim onward. * The function does not split a row across chunks. */ -nx::core::usize computeRowsForByteTarget(const nx::core::ShapeType& dims, nx::core::usize firstRowDim, nx::core::usize unitBytes, nx::core::usize targetBytes, nx::core::usize maxRows) +nx::core::usize computeRowsForByteTarget(nonstd::span dims, nx::core::usize firstRowDim, nx::core::usize unitBytes, nx::core::usize targetBytes, nx::core::usize maxRows) { nx::core::usize rowBytes = unitBytes; for(nx::core::usize i = firstRowDim; i < dims.size(); ++i) @@ -34,12 +38,18 @@ nx::core::usize computeRowsForByteTarget(const nx::core::ShapeType& dims, nx::co namespace nx::core::HDF5 { -ShapeType computeChunkShape(const ShapeType& dims, usize numComponents, usize elementByteSize, const ChunkShapeOptions& opts) +bool ComputeChunkShapeInto(nonstd::span dims, usize numComponents, usize elementByteSize, const ChunkShapeOptions& opts, nonstd::span chunk, + nonstd::span suffixBytes) noexcept { + if(chunk.size() < dims.size() || (opts.regime != ChunkShapeRegime::PinSlowestDim && suffixBytes.size() < dims.size())) + { + return false; + } if(dims.empty()) { - return {}; + return true; } + std::ranges::copy(dims, chunk.begin()); // Fold the full component extent into one tuple so tuple-only dimensions still // produce a physical-byte target. @@ -49,7 +59,6 @@ ShapeType computeChunkShape(const ShapeType& dims, usize numComponents, usize el { // Rank three or greater pins the slowest dimension to one and bands the next // dimension. Lower ranks band the slowest dimension. Inner dimensions stay full. - ShapeType chunk(dims); if(dims.size() >= 3) { chunk[0] = 1; @@ -59,13 +68,11 @@ ShapeType computeChunkShape(const ShapeType& dims, usize numComponents, usize el { chunk[0] = computeRowsForByteTarget(dims, /*firstRowDim=*/1, unitBytes, opts.targetBytes, dims[0]); } - return chunk; + return true; } // suffixBytes[i] is the byte cost of one index step in dimension i. // Calculate it from the innermost dimension to support the outermost-first walk. - ShapeType chunk(dims); - ShapeType suffixBytes(dims.size()); usize inner = unitBytes; for(usize i = dims.size(); i-- > 0;) { @@ -86,6 +93,16 @@ ShapeType computeChunkShape(const ShapeType& dims, usize numComponents, usize el chunk[i] = rows; break; } + return true; +} + +ShapeType computeChunkShape(const ShapeType& dims, usize numComponents, usize elementByteSize, const ChunkShapeOptions& opts) +{ + ShapeType chunk(dims.size()); + ShapeType scratch(opts.regime != ChunkShapeRegime::PinSlowestDim ? dims.size() : 0); + const bool valid = ComputeChunkShapeInto(dims, numComponents, elementByteSize, opts, chunk, scratch); + assert(valid); + (void)valid; return chunk; } diff --git a/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp b/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp index 8b9d99d43c..f3c1b195f3 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp @@ -4,6 +4,8 @@ #include "simplnx/Common/Types.hpp" #include "simplnx/simplnx_export.hpp" +#include + namespace nx::core::HDF5 { @@ -61,6 +63,21 @@ struct ChunkShapeOptions ChunkShapeRegime regime = ChunkShapeRegime::BundleOuterSlabs; }; +/** + * @brief Computes the chunk policy in caller-owned storage without allocation. + * @param dims Supplies tuple or full dataset dimensions. + * @param numComponents Supplies the trailing component count. + * @param elementByteSize Supplies bytes per scalar. + * @param opts Selects the target and regime. + * @param chunkShape Receives the first dims.size() chunk extents. + * @param suffixBytes Supplies dims.size() scratch elements for BundleOuterSlabs; otherwise it can be empty. + * @return False for insufficient storage, before either output changes. + * @pre Input and output ranges do not overlap. Scalar parameters are nonzero and intermediate products fit usize. + * @note Empty dimensions touch no storage. Unused output and scratch tails remain unchanged. + */ +SIMPLNX_EXPORT bool ComputeChunkShapeInto(nonstd::span dims, usize numComponents, usize elementByteSize, const ChunkShapeOptions& opts, nonstd::span chunkShape, + nonstd::span suffixBytes) noexcept; + /** * @brief Computes a byte-targeted row-band chunk shape. * @param dims Dataset dimensions, slowest-varying first. diff --git a/src/simplnx/Utilities/Parsing/HDF5/DeflateEligibility.cpp b/src/simplnx/Utilities/Parsing/HDF5/DeflateEligibility.cpp index 87d3130856..0998905ce7 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/DeflateEligibility.cpp +++ b/src/simplnx/Utilities/Parsing/HDF5/DeflateEligibility.cpp @@ -1,4 +1,6 @@ #include "simplnx/Utilities/Parsing/HDF5/DeflateEligibility.hpp" +#include "simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp" +#include #include "simplnx/Utilities/Parsing/HDF5/H5Support.hpp" @@ -25,21 +27,21 @@ bool probeSingleDeflateEligibility(hid_t datasetId, usize elementSize, hid_t mem // Require one deflate filter. The captured level lets raw writers match the // dataset creation property list. - hid_t dcpl = H5Dget_create_plist(datasetId); + hid_t dcpl = SIMPLNX_OBSERVE_METADATA_CALL(MetadataLayoutCall, H5Dget_create_plist(datasetId)); if(dcpl < 0) { return false; } - const int nFilters = H5Pget_nfilters(dcpl); + const int nFilters = SIMPLNX_OBSERVE_METADATA_CALL(MetadataFilterCall, H5Pget_nfilters(dcpl)); bool deflateOnly = false; if(nFilters == 1) { unsigned int flags = 0; size_t cdNelmts = 16; // Input capacity and output client-value count. - unsigned int cdValues[16] = {0}; - char name[64] = {0}; + std::array cdValues{}; + std::array name{}; unsigned int filterConfig = 0; - const H5Z_filter_t filterId = H5Pget_filter2(dcpl, 0, &flags, &cdNelmts, cdValues, sizeof(name), name, &filterConfig); + const H5Z_filter_t filterId = SIMPLNX_OBSERVE_METADATA_CALL(MetadataFilterCall, H5Pget_filter2(dcpl, 0, &flags, &cdNelmts, cdValues.data(), name.size(), name.data(), &filterConfig)); deflateOnly = (filterId == H5Z_FILTER_DEFLATE); if(deflateOnly && cdNelmts >= 1 && deflateLevelOut != nullptr) { @@ -53,13 +55,14 @@ bool probeSingleDeflateEligibility(hid_t datasetId, usize elementSize, hid_t mem } // Raw bytes require the complete memory representation, including width, signedness, and precision. - const hid_t dtype = H5Dget_type(datasetId); + const hid_t dtype = SIMPLNX_OBSERVE_METADATA_CALL(MetadataTypeCall, H5Dget_type(datasetId)); if(dtype < 0) { return false; } - const bool identical = elementSize > 0 && memoryTypeId >= 0 && H5Tget_size(memoryTypeId) == elementSize && H5Tequal(dtype, memoryTypeId) > 0; - const H5T_order_t order = identical && elementSize > 1 ? H5Tget_order(dtype) : H5T_ORDER_ERROR; + const bool identical = elementSize > 0 && memoryTypeId >= 0 && SIMPLNX_OBSERVE_METADATA_CALL(MetadataTypeCall, H5Tget_size(memoryTypeId)) == elementSize && + SIMPLNX_OBSERVE_METADATA_CALL(MetadataTypeCall, H5Tequal(dtype, memoryTypeId)) > 0; + const H5T_order_t order = identical && elementSize > 1 ? SIMPLNX_OBSERVE_METADATA_CALL(MetadataTypeCall, H5Tget_order(dtype)) : H5T_ORDER_ERROR; H5Tclose(dtype); const H5T_order_t hostOrder = hostIsLittleEndian() ? H5T_ORDER_LE : H5T_ORDER_BE; return identical && (elementSize == 1 || order == hostOrder); diff --git a/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.cpp b/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.cpp index 3e90edd0f2..0684e73cd3 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.cpp +++ b/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.cpp @@ -1,5 +1,6 @@ #include "DatasetIO.hpp" +#include "simplnx/Common/ScopeGuard.hpp" #include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/Utilities/Parsing/HDF5/ChunkIndex.hpp" #include "simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp" @@ -12,6 +13,7 @@ #include "H5Dpublic.h" #include "H5Fpublic.h" +#include "H5Ipublic.h" #include "H5Spublic.h" #include "H5Tpublic.h" @@ -22,6 +24,7 @@ #include #include #include +#include #include using namespace nx::core; @@ -195,7 +198,7 @@ void DatasetIO::close() { const hid_t selfId = getId(); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); H5Dclose(selfId); } setId(0); @@ -215,7 +218,7 @@ hid_t DatasetIO::open() const const std::string namePath = getNamePath(); hid_t id = H5I_INVALID_HID; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); HDF_ERROR_HANDLER_OFF /* Does not matter what the 'id' is, we are accepting that value. */ id = H5Dopen(parentId, namePath.c_str(), H5P_DEFAULT); HDF_ERROR_HANDLER_ON @@ -237,7 +240,7 @@ hid_t DatasetIO::createOrOpenDataset(IdType typeId, IdType dataspaceId, IdType p const hid_t parentId = getParentId(); const std::string namePath = getNamePath(); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); HDF_ERROR_HANDLER_OFF setId(H5Dopen(parentId, namePath.c_str(), H5P_DEFAULT)); HDF_ERROR_HANDLER_ON @@ -352,7 +355,7 @@ hid_t DatasetIO::getTypeId() const // getId() self-locks (it may lazily open the dataset); resolve it before the leaf-locked // bare H5Dget_type. auto identifier = getId(); - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); return H5Dget_type(identifier); } @@ -360,7 +363,7 @@ hid_t DatasetIO::getClassType() const { // getId() can open the dataset. Resolve it before the non-recursive HDF5 API lock. const hid_t identifier = getId(); - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const hid_t typeId = H5Dget_type(identifier); if(typeId < 0) { @@ -376,7 +379,7 @@ size_t DatasetIO::getTypeSize() const { // getId() can open the dataset. Resolve it before the non-recursive HDF5 API lock. const hid_t identifier = getId(); - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const hid_t typeId = H5Dget_type(identifier); if(typeId < 0) { @@ -448,7 +451,7 @@ Result DatasetIO::getDataType() const hid_t typeId = -1; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); typeId = H5Dget_type(datasetId); const H5T_class_t classType = H5Tget_class(typeId); if(classType == H5T_COMPOUND) @@ -459,7 +462,7 @@ Result DatasetIO::getDataType() const } const Type type = getTypeFromId(typeId); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); H5Tclose(typeId); } if(type == Type::unknown) @@ -539,7 +542,7 @@ std::string DatasetIO::readAsString() const // only after releasing this nonrecursive lock. bool isVariableString = false; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const hid_t typeID = H5Dget_type(datasetId); isVariableString = (H5Tis_variable_str(typeID) == 1); H5Tclose(typeID); @@ -561,7 +564,7 @@ std::string DatasetIO::readAsString() const // Fixed-length string: read the raw bytes under one leaf lock. std::string data; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const hid_t typeID = H5Dget_type(datasetId); const hsize_t size = H5Dget_storage_size(datasetId); std::vector buffer(static_cast(size + 1), 0x00); // Allocate and zero the array @@ -592,7 +595,7 @@ std::vector DatasetIO::readAsVectorOfStrings() const // One leaf critical section spanning every bare H5 call below (type/space query, // memory-type build, vlen read, and reclaim). open()/isValid() were resolved // above and no other self-locking helper runs inside this scope. - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); hid_t typeID = H5Dget_type(datasetId); if(typeID >= 0) @@ -733,7 +736,7 @@ nx::core::Result<> DatasetIO::readIntoSpan(nonstd::span data) const hid_t dataType = H5I_INVALID_HID; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); dataType = HdfTypeForPrimitive(); } if(dataType == -1) @@ -766,7 +769,7 @@ nx::core::Result<> DatasetIO::readIntoSpan(nonstd::span data) const std::filesystem::path codecFilePath = getFilePath(); if(codecFilePath.empty()) { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const ssize_t nameLen = H5Fget_name(datasetId, nullptr, 0); if(nameLen > 0) { @@ -863,7 +866,7 @@ Result<> DatasetIO::readIntoSpan(nonstd::span data, const std::optional hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); dataType = HdfTypeForPrimitive(); } if(dataType == -1) @@ -877,7 +880,7 @@ Result<> DatasetIO::readIntoSpan(nonstd::span data, const std::optional hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); hid_t fileSpaceId = H5Dget_space(datasetId); if(fileSpaceId < 0) { @@ -1127,7 +1130,7 @@ std::vector DatasetIO::getChunkDimensions() const const hid_t selfId = open(); const usize numDims = getDimensions().size(); - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); auto propertyListId = H5Dget_create_plist(selfId); H5D_layout_t layout = H5Pget_layout(propertyListId); std::vector result; @@ -1152,7 +1155,7 @@ std::vector DatasetIO::getDimensions() const std::vector dims; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); auto dataspaceId = H5Dget_space(selfId); if(dataspaceId >= 0) { @@ -1193,7 +1196,7 @@ Result<> DatasetIO::writeSpan(const DimsType& dims, nonstd::span values int32_t rank = static_cast(dims.size()); hid_t dataType = H5I_INVALID_HID; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); dataType = HdfTypeForPrimitive(); } if(dataType == -1) @@ -1208,7 +1211,7 @@ Result<> DatasetIO::writeSpan(const DimsType& dims, nonstd::span values // another public wrapper call. hid_t dataspaceId = H5I_INVALID_HID; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); dataspaceId = H5Screate_simple(rank, hDims.data(), nullptr); } @@ -1260,7 +1263,7 @@ Result<> DatasetIO::writeSpan(const DimsType& dims, nonstd::span values // The serial fallback locks only H5Dwrite. Error formatting stays outside. const void* data = static_cast(values.data()); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); error = H5Dwrite(datasetId, dataType, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); } if(error < 0) @@ -1276,7 +1279,7 @@ Result<> DatasetIO::writeSpan(const DimsType& dims, nonstd::span values if(dcpl != H5P_DEFAULT) { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); H5Pclose(dcpl); } } @@ -1318,7 +1321,7 @@ Result<> DatasetIO::createEmptyDataset(const DimsType& dims) std::transform(dims.begin(), dims.end(), hDims.begin(), [](DimsType::value_type x) { return static_cast(x); }); hid_t dataspaceId = H5I_INVALID_HID; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); dataspaceId = H5Screate_simple(static_cast(hDims.size()), hDims.data(), nullptr); } if(dataspaceId < 0) @@ -1331,7 +1334,7 @@ Result<> DatasetIO::createEmptyDataset(const DimsType& dims) if(dcplResult.invalid()) { { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); H5Sclose(dataspaceId); } return MakeErrorResult(dcplResult.errors().front().code, fmt::format("createEmptyDataset error: unable to build dataset creation property list for dataset '{}' in file '{}': {}", getNamePath(), @@ -1342,7 +1345,7 @@ Result<> DatasetIO::createEmptyDataset(const DimsType& dims) // createOrOpenDataset() locks itself. Close local properties in a later leaf scope. auto datasetId = createOrOpenDataset(dataspaceId, dcpl); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); H5Sclose(dataspaceId); if(dcpl != H5P_DEFAULT) { @@ -1360,78 +1363,243 @@ Result<> DatasetIO::createEmptyDataset(const DimsType& dims) template Result<> DatasetIO::writeSpanHyperslab(nonstd::span values, const std::vector& start, const std::vector& count) { + std::string datasetPath = getNamePath(); + std::string filePathString = getFilePath().string(); if(!isValid()) { - return MakeErrorResult(-506, fmt::format("Cannot open HDF5 data at {} / {}", getFilePath().string(), getNamePath())); + return MakeErrorResult(-506, fmt::format("Cannot write dataset '{}' in '{}': the dataset wrapper is not open.", datasetPath, filePathString)); } - hid_t dataType = HdfTypeForPrimitive(); - if(dataType == -1) - { - return MakeErrorResult(-1010, "writeSpanHyperslab error: Unsupported data type."); - } - - // open() locks itself before the leaf hyperslab critical section. + // Wrapper operations can lock. Resolve the handle and diagnostic paths before the transfer lock. const hid_t datasetId = open(); - - int errorCode = 0; - herr_t writeError = 0; + // An open dataset can have no links. Keep its stored name when the native name is unavailable. { - std::lock_guard hdf5Lock(Support::ApiLock()); - hid_t fileSpaceId = H5Dget_space(datasetId); - if(fileSpaceId < 0) + const std::lock_guard hdf5Lock(Support::ApiLock()); + const ssize_t nameLength = H5Iget_name(datasetId, nullptr, 0); + if(nameLength > 0 && std::cmp_less(nameLength, std::numeric_limits::max())) { - errorCode = -1011; + std::string nameBuffer(static_cast(nameLength) + 1, '\0'); + const ssize_t copiedLength = H5Iget_name(datasetId, nameBuffer.data(), nameBuffer.size()); + if(copiedLength > 0 && copiedLength <= nameLength) + { + nameBuffer.resize(static_cast(copiedLength)); + datasetPath = std::move(nameBuffer); + } } - else + } + if(datasetPath.empty()) + { + datasetPath = ""; + } + // Group-created wrappers can lack a stored file path. Query the owning file in a separate leaf lock. + if(filePathString.empty()) + { + const std::lock_guard hdf5Lock(Support::ApiLock()); + const ssize_t nameLength = H5Fget_name(datasetId, nullptr, 0); + if(nameLength > 0 && std::cmp_less(nameLength, std::numeric_limits::max())) { - // macOS uses a different hsize_t underlying type. Copy offsets to avoid pointer casts. -#if defined(__APPLE__) - std::vector startVec(start.begin(), start.end()); - std::vector countVec(count.begin(), count.end()); - if(H5Sselect_hyperslab(fileSpaceId, H5S_SELECT_SET, startVec.data(), NULL, countVec.data(), NULL) < 0) -#else - if(H5Sselect_hyperslab(fileSpaceId, H5S_SELECT_SET, start.data(), NULL, count.data(), NULL) < 0) -#endif + std::string nameBuffer(static_cast(nameLength) + 1, '\0'); + const ssize_t copiedLength = H5Fget_name(datasetId, nameBuffer.data(), nameBuffer.size()); + if(copiedLength > 0 && copiedLength <= nameLength) { - errorCode = -1012; + nameBuffer.resize(static_cast(copiedLength)); + filePathString = std::move(nameBuffer); } - else + } + } + if(filePathString.empty()) + { + filePathString = ""; + } + + enum class Failure + { + None, + Native, + UnsupportedType, + UnsupportedDataspace, + Rank, + CoordinateRange, + Bounds, + ElementOverflow, + ByteOverflow, + BufferSize, + NullBuffer + }; + + uint64 elementLimit = std::numeric_limits::max(); + if constexpr(std::cmp_less(std::numeric_limits::digits, std::numeric_limits::digits)) + { + elementLimit = static_cast(std::numeric_limits::max()); + } + if constexpr(std::cmp_less(std::numeric_limits::digits, std::numeric_limits::digits)) + { + elementLimit = std::min(elementLimit, static_cast(std::numeric_limits::max())); + } + + int32 nativeErrorCode = -1011; + const char* nativeOperationPtr = nullptr; + hid_t nativeStatus = 0; + H5S_class_t spaceType = H5S_NO_CLASS; + int rank = 0; + usize axisIdx = 0; + hsize_t axisDimension = 0; + uint64 selectedElements = 1; + Failure failure = Failure::None; + { + const std::lock_guard hdf5Lock(Support::ApiLock()); + // Early returns close owned dataspaces before the nonrecursive lock is released. + failure = [&]() -> Failure { + const hid_t dataType = HdfTypeForPrimitive(); + if(dataType == -1) + { + return Failure::UnsupportedType; + } + + nativeOperationPtr = "H5Dget_space"; + nativeStatus = H5Dget_space(datasetId); + if(nativeStatus < 0) + { + return Failure::Native; + } + const hid_t fileSpaceId = nativeStatus; + auto fileSpaceGuard = MakeScopeGuard([fileSpaceId]() noexcept { H5Sclose(fileSpaceId); }); + + nativeOperationPtr = "H5Sget_simple_extent_type"; + spaceType = H5Sget_simple_extent_type(fileSpaceId); + if(spaceType == H5S_NO_CLASS) + { + nativeStatus = static_cast(spaceType); + return Failure::Native; + } + if(spaceType != H5S_SIMPLE) + { + return Failure::UnsupportedDataspace; + } + + nativeOperationPtr = "H5Sget_simple_extent_ndims"; + rank = H5Sget_simple_extent_ndims(fileSpaceId); + if(rank < 0) + { + nativeStatus = rank; + return Failure::Native; + } + if(start.size() != static_cast(rank) || count.size() != static_cast(rank)) + { + return Failure::Rank; + } + + std::vector dimensions(static_cast(rank)); + nativeOperationPtr = "H5Sget_simple_extent_dims"; + nativeStatus = H5Sget_simple_extent_dims(fileSpaceId, dimensions.data(), nullptr); + if(nativeStatus < 0) + { + return Failure::Native; + } + std::vector nativeStart(start.size()); + std::vector nativeCount(count.size()); + bool emptySelection = false; + for(axisIdx = 0; axisIdx < count.size(); ++axisIdx) { - // Match the memory dataspace to the selected extent. - std::vector memDims(count.begin(), count.end()); - hid_t memSpaceId = H5Screate_simple(static_cast(memDims.size()), memDims.data(), nullptr); - if(memSpaceId < 0) + axisDimension = dimensions[axisIdx]; + if(!std::in_range(start[axisIdx]) || !std::in_range(count[axisIdx])) { - errorCode = -1013; + return Failure::CoordinateRange; } - else + nativeStart[axisIdx] = static_cast(start[axisIdx]); + nativeCount[axisIdx] = static_cast(count[axisIdx]); + if(nativeStart[axisIdx] > axisDimension || nativeCount[axisIdx] > axisDimension - nativeStart[axisIdx]) { - writeError = H5Dwrite(datasetId, dataType, memSpaceId, fileSpaceId, H5P_DEFAULT, values.data()); - if(writeError < 0) - { - errorCode = -1014; - } - H5Sclose(memSpaceId); + return Failure::Bounds; } + emptySelection = emptySelection || count[axisIdx] == 0; } - H5Sclose(fileSpaceId); - } + + // Validate every axis before empty success; an empty selection needs no buffer or product calculation. + if(emptySelection) + { + return Failure::None; + } + for(axisIdx = 0; axisIdx < count.size(); ++axisIdx) + { + if(selectedElements > elementLimit / count[axisIdx]) + { + return Failure::ElementOverflow; + } + selectedElements *= count[axisIdx]; + } + if(std::cmp_greater(selectedElements, std::numeric_limits::max() / sizeof(T))) + { + return Failure::ByteOverflow; + } + if(std::cmp_less(values.size(), selectedElements)) + { + return Failure::BufferSize; + } + if(values.data() == nullptr) + { + return Failure::NullBuffer; + } + + nativeErrorCode = -1012; + nativeOperationPtr = "H5Sselect_hyperslab"; + nativeStatus = H5Sselect_hyperslab(fileSpaceId, H5S_SELECT_SET, nativeStart.data(), nullptr, nativeCount.data(), nullptr); + if(nativeStatus < 0) + { + return Failure::Native; + } + + nativeErrorCode = -1013; + nativeOperationPtr = "H5Screate_simple"; + nativeStatus = H5Screate_simple(rank, nativeCount.data(), nullptr); + if(nativeStatus < 0) + { + return Failure::Native; + } + const hid_t memSpaceId = nativeStatus; + auto memSpaceGuard = MakeScopeGuard([memSpaceId]() noexcept { H5Sclose(memSpaceId); }); + + nativeErrorCode = -1014; + nativeOperationPtr = "H5Dwrite"; + nativeStatus = H5Dwrite(datasetId, dataType, memSpaceId, fileSpaceId, H5P_DEFAULT, values.data()); + return nativeStatus < 0 ? Failure::Native : Failure::None; + }(); } - switch(errorCode) + // Error formatting stays outside the lock, after all owned dataspaces close. + switch(failure) { - case 0: + case Failure::None: return {}; - case -1011: - return MakeErrorResult(-1011, "writeSpanHyperslab error: Unable to open the dataspace."); - case -1012: - return MakeErrorResult(-1012, "writeSpanHyperslab error: Unable to select hyperslab."); - case -1013: - return MakeErrorResult(-1013, "writeSpanHyperslab error: Unable to create memory dataspace."); - default: - return MakeErrorResult(-1014, fmt::format("writeSpanHyperslab error: H5Dwrite failed with error {}", writeError)); + case Failure::Native: + return MakeErrorResult(nativeErrorCode, fmt::format("Cannot write dataset '{}' in '{}': {} failed with status {}.", datasetPath, filePathString, nativeOperationPtr, nativeStatus)); + case Failure::UnsupportedType: + return MakeErrorResult(-1010, fmt::format("Cannot write dataset '{}' in '{}': the input scalar type is not supported by HDF5.", datasetPath, filePathString)); + case Failure::UnsupportedDataspace: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': hyperslab writes do not support {} dataspaces. Use a full-dataset write for scalar values.", datasetPath, + filePathString, spaceType == H5S_SCALAR ? "scalar" : "null")); + case Failure::Rank: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': dataset rank {} requires matching start and count ranks, but received {} start offsets and {} counts.", datasetPath, + filePathString, rank, start.size(), count.size())); + case Failure::CoordinateRange: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': axis {} has start {}, count {}, and dimension {}. Start and count must fit the HDF5 coordinate limit {}.", + datasetPath, filePathString, axisIdx, start[axisIdx], count[axisIdx], axisDimension, std::numeric_limits::max())); + case Failure::Bounds: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': axis {} has start {}, count {}, and dimension {}. The selection must stay within the dataset extent.", datasetPath, + filePathString, axisIdx, start[axisIdx], count[axisIdx], axisDimension)); + case Failure::ElementOverflow: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': selected element count overflow at axis {}. Prefix {} multiplied by count {} exceeds the element limit {}.", + datasetPath, filePathString, axisIdx, selectedElements, count[axisIdx], elementLimit)); + case Failure::ByteOverflow: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': selected byte count overflow. {} elements with {} bytes per value exceed the buffer byte limit {}.", datasetPath, + filePathString, selectedElements, sizeof(T), std::numeric_limits::max())); + case Failure::BufferSize: + return MakeErrorResult( + -1012, fmt::format("Cannot write dataset '{}' in '{}': the selection requires {} values, but the input span contains {}.", datasetPath, filePathString, selectedElements, values.size())); + case Failure::NullBuffer: + return MakeErrorResult(-1012, fmt::format("Cannot write dataset '{}' in '{}': the selection requires {} values, but the input buffer is null.", datasetPath, filePathString, selectedElements)); } + return {}; } template @@ -1442,7 +1610,7 @@ nx::core::Result DatasetIO::initChunkedDataset(const DimsType& // Create the dataspace before the separately locked dataset open or create call. { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); dataInfo.dataspaceId = H5Screate_simple(h5Dims.size(), h5DimsVec.data(), nullptr); } if(dataInfo.dataspaceId < 0) @@ -1478,7 +1646,7 @@ hid_t DatasetIO::CreateH5DatasetChunkProperties(const DimsType& chunkDims) { std::vector hDims(chunkDims.size()); std::transform(chunkDims.begin(), chunkDims.end(), hDims.begin(), [](DimsType::value_type x) { return static_cast(x); }); - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); auto cparms = H5Pcreate(H5P_DATASET_CREATE); auto status = H5Pset_chunk(cparms, hDims.size(), hDims.data()); if(status < 0) @@ -1512,7 +1680,7 @@ nx::core::Result<> DatasetIO::closeChunkedDataset(const ChunkedDataInfo& dataset }*/ herr_t error = 0; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); error = H5Sclose(datasetInfo.dataspaceId); } if(error < 0) @@ -1561,7 +1729,7 @@ nx::core::Result<> DatasetIO::readChunk(const ChunkedDataInfo& chunkInfo, const std::vector offsetVec(offset.begin(), offset.end()); std::vector chunkShapeVec(chunkShape.begin(), chunkShape.end()); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const hid_t plistId = H5Dget_create_plist(h5Id); if(plistId > 0) { @@ -1655,7 +1823,7 @@ Result<> DatasetIO::writeChunk(const ChunkedDataInfo& chunkInfo, const DimsType& // Open chunk handles permit one leaf lock around selection, write, and cleanup. { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); error = H5Sselect_hyperslab(dataspaceId, H5S_SELECT_SET, offsetVec.data(), NULL, trueChunkShapeVec.data(), NULL); // Create memory dataspace for the hyperslab @@ -1716,7 +1884,7 @@ nx::core::Result<> DatasetIO::writeString(const std::string& text) const hid_t parentId = getParentId(); const std::string namePath = getNamePath(); { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); /* create a string data type */ hid_t typeId; @@ -1783,7 +1951,7 @@ nx::core::Result<> DatasetIO::writeVectorOfStrings(const std::vector dims = {text.size()}; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); if((dataspaceID = H5Screate_simple(static_cast(dims.size()), dims.data(), nullptr)) >= 0) { dims[0] = 1; @@ -1900,7 +2068,7 @@ std::string DatasetIO::getFilterName() const const hid_t selfId = getId(); std::string filterNames; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); const hid_t cpListId = H5Dget_create_plist(selfId); const int numFilters = H5Pget_nfilters(cpListId); for(int j = 0; j < numFilters; ++j) diff --git a/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.hpp b/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.hpp index 96f4ab8a6e..50a21b127e 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/IO/DatasetIO.hpp @@ -245,7 +245,7 @@ class SIMPLNX_EXPORT DatasetIO : public ObjectIO * @param dims Specifies N-dimensional dataset shape. * @return Shape or HDF5 creation error, or success. * - * OOC writers call this before region-by-region writeSpanHyperslab() transfers. + * Use writeSpanHyperslab() to transfer selected regions after creation. */ template nx::core::Result<> createEmptyDataset(const DimsType& dims); @@ -253,11 +253,16 @@ class SIMPLNX_EXPORT DatasetIO : public ObjectIO /** * @brief Writes one typed hyperslab to an existing dataset. * @tparam T Specifies the scalar type. - * @param values Provides the selected element count. - * @param start Specifies N-dimensional start offsets. - * @param count Specifies N-dimensional extents. - * @return Selection, shape, or HDF5 write error, or success. - * @pre createEmptyDataset() or writeSpan() created the dataset. + * @param values Provides at least the selected element count; unused trailing values are ignored. + * @param start Specifies one start offset per dataset axis, at or within its extent. + * @param count Specifies one extent per axis, within the dimension after its start offset. + * @return Success, or a selection, buffer, dataspace, or native HDF5 error. + * @pre This wrapper already has an open dataset identifier. + * + * Both vectors must match the dataset rank. All axes are validated before an empty simple selection succeeds without a transfer. + * A start offset can equal its dimension only when that axis has zero count. + * Nonempty selections require nonnull storage and representable element and byte counts. + * Scalar and null dataspaces do not support hyperslab writes. */ template nx::core::Result<> writeSpanHyperslab(nonstd::span values, const std::vector& start, const std::vector& count); diff --git a/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.cpp b/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.cpp index 87c6a91dc4..3af8af1b14 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.cpp +++ b/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.cpp @@ -1,4 +1,11 @@ #include "simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp" +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Types.hpp" +#include +#include +#include +#include +#include #include "simplnx/Utilities/Parsing/HDF5/ChunkIndex.hpp" #include "simplnx/Utilities/Parsing/HDF5/DeflateEligibility.hpp" @@ -30,9 +37,281 @@ #include #include #include +#include namespace nx::core::HDF5 { +#if SIMPLNX_BUILD_TESTS +namespace +{ +CodecIoObserverForTesting CodecIoObserver; +} + +namespace +{ +std::atomic LegacyOutstanding{0}; +std::atomic LegacyTasks{0}; +std::atomic LegacyErrorAggregations{0}; +std::atomic LegacyIntegrityFailures{0}; +std::atomic LegacyUnobservedReturns{0}; +thread_local testing_detail::LegacyPayloadRecordForTesting PendingLegacyPayload; +thread_local LegacyPayloadTicketForTesting* LegacyAdmissionTicket = nullptr; +thread_local uint64 LegacyThreadOutstanding = 0; + +/** + * @brief Discharges a recorded receipt without touching its former byte storage. + * @param record Supplies the unique charged observation. + * @param event Selects destruction or cache transfer. + */ +void releaseLegacyRecord(testing_detail::LegacyPayloadRecordForTesting& record, CodecIoEventForTesting event) noexcept +{ + if(record.charged) + { + ObserveCodecIoForTesting(*record.file, record.dataset, event, record.capacity, static_cast(record.kind)); + LegacyOutstanding.fetch_sub(1, std::memory_order_relaxed); + --LegacyThreadOutstanding; + record = {}; + } +} + +/** + * @brief Invalidates the observation when one worker leaves a pending return. + */ +void settleStrandedLegacyReceipt() noexcept +{ + if(PendingLegacyPayload.charged) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + releaseLegacyRecord(PendingLegacyPayload, CodecIoEventForTesting::LegacyReleased); + } +} +} // namespace + +CodecOwnershipStateForTesting GetCodecOwnershipStateForTesting() noexcept +{ + return {LegacyOutstanding.load(std::memory_order_relaxed), LegacyTasks.load(std::memory_order_relaxed), LegacyIntegrityFailures.load(std::memory_order_relaxed), + LegacyUnobservedReturns.load(std::memory_order_relaxed), LegacyErrorAggregations.load(std::memory_order_relaxed)}; +} + +BoundedReadTaskScopeForTesting::BoundedReadTaskScopeForTesting() noexcept +: m_Active(CodecIoObserver.callback != nullptr || CodecIoObserver.boundedFault != nullptr) +{ + if(m_Active) + { + LegacyTasks.fetch_add(1, std::memory_order_relaxed); + } +} + +BoundedReadTaskScopeForTesting::~BoundedReadTaskScopeForTesting() noexcept +{ + if(m_Active) + { + LegacyTasks.fetch_sub(1, std::memory_order_relaxed); + } +} + +bool InjectCodecFaultForTesting(const std::filesystem::path& file, std::string_view dataset, BoundedFaultPointForTesting point) noexcept +{ + return CodecIoObserver.boundedFault != nullptr && CodecIoObserver.boundedFault(CodecIoObserver.context, file, dataset, point); +} + +LegacyPayloadTicketForTesting::~LegacyPayloadTicketForTesting() noexcept +{ + releaseDestroyed(); +} + +LegacyPayloadTicketForTesting::LegacyPayloadTicketForTesting(LegacyPayloadTicketForTesting&& other) noexcept +: m_Record(std::exchange(other.m_Record, {})) +{ +} + +void LegacyPayloadTicketForTesting::acquire(const std::filesystem::path& file, std::string_view dataset, const std::vector& bytes, LegacyPayloadKindForTesting kind) noexcept +{ + if(CodecIoObserver.callback == nullptr || bytes.capacity() == 0) + { + return; + } + if(m_Record.charged) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + return; + } + m_Record = {&file, dataset, bytes.data(), bytes.capacity(), kind, true}; + LegacyOutstanding.fetch_add(1, std::memory_order_relaxed); + ++LegacyThreadOutstanding; + ObserveCodecIoForTesting(file, dataset, CodecIoEventForTesting::LegacyAcquired, bytes.capacity(), static_cast(kind)); +} + +void LegacyPayloadTicketForTesting::consumeReturn(const std::vector& bytes, bool allowUnobserved) noexcept +{ + if(m_Record.charged) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + return; + } + if(!PendingLegacyPayload.charged) + { + if(CodecIoObserver.callback != nullptr && !bytes.empty()) + { + (allowUnobserved ? LegacyUnobservedReturns : LegacyIntegrityFailures).fetch_add(1, std::memory_order_relaxed); + } + return; + } + if(PendingLegacyPayload.data != bytes.data() || PendingLegacyPayload.capacity != bytes.capacity()) + { + settleStrandedLegacyReceipt(); + return; + } + m_Record = std::exchange(PendingLegacyPayload, {}); +} + +void LegacyPayloadTicketForTesting::replaceDestroyedFromReturn(const std::vector& bytes, bool allowUnobserved) noexcept +{ + auto old = std::exchange(m_Record, {}); + consumeReturn(bytes, allowUnobserved); + releaseLegacyRecord(old, CodecIoEventForTesting::LegacyReleased); +} + +void LegacyPayloadTicketForTesting::forwardReturn() noexcept +{ + if(!m_Record.charged) + { + return; + } + if(PendingLegacyPayload.charged) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + return; + } + if(m_Record.kind != LegacyPayloadKindForTesting::Returned) + { + ObserveCodecIoForTesting(*m_Record.file, m_Record.dataset, CodecIoEventForTesting::LegacyToReturned, m_Record.capacity, static_cast(m_Record.kind)); + m_Record.kind = LegacyPayloadKindForTesting::Returned; + } + PendingLegacyPayload = std::exchange(m_Record, {}); +} + +void LegacyPayloadTicketForTesting::releaseDestroyed() noexcept +{ + releaseLegacyRecord(m_Record, CodecIoEventForTesting::LegacyReleased); +} + +void LegacyPayloadTicketForTesting::transferToCache() noexcept +{ + releaseLegacyRecord(m_Record, CodecIoEventForTesting::LegacyTransferred); +} + +bool LegacyPayloadTicketForTesting::matches(const std::vector& bytes) const noexcept +{ + return m_Record.charged && !bytes.empty() && m_Record.data == bytes.data() && m_Record.capacity == bytes.capacity(); +} + +bool LegacyPayloadTicketForTesting::charged() const noexcept +{ + return m_Record.charged; +} + +LegacyPayloadTaskScopeForTesting::LegacyPayloadTaskScopeForTesting() noexcept +: m_Active(CodecIoObserver.callback != nullptr) +, m_ThreadOutstandingAtEntry(LegacyThreadOutstanding) +, m_AdmissionAtEntry(LegacyAdmissionTicket) +{ + if(m_Active) + { + settleStrandedLegacyReceipt(); + m_ThreadOutstandingAtEntry = LegacyThreadOutstanding; + LegacyTasks.fetch_add(1, std::memory_order_relaxed); + } +} + +LegacyPayloadTaskScopeForTesting::~LegacyPayloadTaskScopeForTesting() noexcept +{ + if(m_Active) + { + settleStrandedLegacyReceipt(); + if(LegacyThreadOutstanding != m_ThreadOutstandingAtEntry || LegacyAdmissionTicket != m_AdmissionAtEntry) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + LegacyAdmissionTicket = m_AdmissionAtEntry; + } + LegacyTasks.fetch_sub(1, std::memory_order_relaxed); + } +} + +void LegacyPayloadTaskScopeForTesting::observeExceptionAggregation() noexcept +{ + if(m_Active) + { + LegacyErrorAggregations.fetch_add(1, std::memory_order_relaxed); + if(LegacyThreadOutstanding != m_ThreadOutstandingAtEntry) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + } + } +} + +namespace +{ +thread_local const std::filesystem::path* MetadataFile = nullptr; +thread_local std::string_view MetadataDataset; +} // namespace + +MetadataIdentityForTesting::MetadataIdentityForTesting(const std::filesystem::path& file, std::string_view dataset) noexcept +: m_PreviousFile(std::exchange(MetadataFile, &file)) +, m_PreviousDataset(std::exchange(MetadataDataset, dataset)) +{ +} + +MetadataIdentityForTesting::~MetadataIdentityForTesting() noexcept +{ + MetadataFile = m_PreviousFile; + MetadataDataset = m_PreviousDataset; +} + +void ObserveMetadataCallForTesting(CodecIoEventForTesting event) noexcept +{ + if(MetadataFile != nullptr) + { + ObserveCodecIoForTesting(*MetadataFile, MetadataDataset, event, 1, 0); + } +} + +LegacyPayloadAdmissionScopeForTesting::LegacyPayloadAdmissionScopeForTesting(LegacyPayloadTicketForTesting& ticket) noexcept +: m_Previous(std::exchange(LegacyAdmissionTicket, &ticket)) +{ +} + +LegacyPayloadAdmissionScopeForTesting::~LegacyPayloadAdmissionScopeForTesting() noexcept +{ + LegacyAdmissionTicket = m_Previous; +} + +LegacyPayloadTicketForTesting* MatchLegacyAdmissionForTesting(const std::vector& bytes) noexcept +{ + return LegacyAdmissionTicket != nullptr && LegacyAdmissionTicket->matches(bytes) ? LegacyAdmissionTicket : nullptr; +} + +CodecIoObserverForTesting SetCodecIoObserverForTesting(CodecIoObserverForTesting observer) noexcept +{ + const CodecIoObserverForTesting previous = CodecIoObserver; + if(LegacyOutstanding.load(std::memory_order_relaxed) != 0 || LegacyTasks.load(std::memory_order_relaxed) != 0) + { + LegacyIntegrityFailures.fetch_add(1, std::memory_order_relaxed); + return previous; + } + CodecIoObserver = observer; + return previous; +} + +void ObserveCodecIoForTesting(const std::filesystem::path& filePath, std::string_view datasetPath, CodecIoEventForTesting event, uint64 inputBytes, uint64 outputBytes) noexcept +{ + if(CodecIoObserver.callback != nullptr) + { + CodecIoObserver.callback(CodecIoObserver.context, filePath, datasetPath, event, inputBytes, outputBytes); + } +} +#endif + namespace { constexpr usize k_IncompressibilityProbeBytes = 4 * 1024; @@ -91,7 +370,7 @@ ParallelChunkCodec::ChunkInfo peekChunkInfoUnlocked(hid_t datasetId, const Exten */ ParallelChunkCodec::ChunkInfo peekChunkInfo(hid_t datasetId, const Extent& bounds, usize componentRank) { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); return peekChunkInfoUnlocked(datasetId, bounds, componentRank); } @@ -165,11 +444,34 @@ ParallelChunkCodec::ParallelChunkCodec(std::filesystem::path filePath, std::stri , m_ElementSize(elementSize) , m_DatasetId(datasetId) { +#if SIMPLNX_BUILD_TESTS + const MetadataIdentityForTesting metadataIdentity(m_FilePath, m_DatasetPath); +#endif m_NumComponents = product(m_ComponentShape); m_NominalChunkElements = product(m_ChunkShape) * m_NumComponents; m_NumChunks = getNumberOfChunks(m_TupleShape, m_ChunkShape); // Probe once for single-deflate eligibility and capture the write deflate level. m_Eligible = probeSingleDeflateEligibility(m_DatasetId, m_ElementSize, memoryTypeId, &m_DeflateLevel); + // This independent tag does not change ordinary eligibility or decoder policy. + { + const std::lock_guard hdf5Lock(Support::ApiLock()); + const std::array nativeTypes = {H5T_NATIVE_INT8, H5T_NATIVE_UINT8, H5T_NATIVE_INT16, H5T_NATIVE_UINT16, H5T_NATIVE_INT32, + H5T_NATIVE_UINT32, H5T_NATIVE_INT64, H5T_NATIVE_UINT64, H5T_NATIVE_FLOAT, H5T_NATIVE_DOUBLE}; + for(const hid_t nativeType : nativeTypes) + { + const htri_t equal = H5Tequal(memoryTypeId, nativeType); + if(equal < 0) + { + m_BoundedTypeQueryFailed = true; + break; + } + if(equal > 0) + { + m_BoundedNativeType = nativeType; + break; + } + } + } } ParallelChunkCodec::~ParallelChunkCodec() @@ -207,6 +509,9 @@ std::vector ParallelChunkCodec::inflateChunk(uint64 flatChunkIndex) c // Reuse one metadata snapshot so this read performs one HDF5 chunk query. const ChunkInfo info = peekChunkInfo(m_DatasetId, bounds, m_ComponentShape.size()); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::ChunkRecordQuery, 1, 0); +#endif if(!info.allocated) { // A sparse chunk has no raw bytes. Best-effort prewarm can skip this distinct error. @@ -221,7 +526,7 @@ std::vector ParallelChunkCodec::getChunkInfos(non chunkInfos.reserve(flatChunkIndices.size()); const std::vector chunksPerDimension = getChunksPerDimension(m_TupleShape, m_ChunkShape); std::vector offset(m_TupleShape.size() + m_ComponentShape.size(), 0); - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); for(uint64 flatChunkIndex : flatChunkIndices) { uint64 remaining = flatChunkIndex; @@ -233,6 +538,9 @@ std::vector ParallelChunkCodec::getChunkInfos(non offset[dimension] = static_cast(chunkCoordinate * m_ChunkShape[dimension]); } chunkInfos.push_back(peekChunkInfoAtOffsetUnlocked(m_DatasetId, offset.data())); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::ChunkRecordQuery, 1, 0); +#endif } return chunkInfos; } @@ -251,7 +559,23 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk { const usize tupleDims = m_TupleShape.size(); +#if SIMPLNX_BUILD_TESTS + usize observedRawCapacity = 0; + bool rawWasObserved = false; + // Declare the guard first so release is observed after the vector is destroyed. + auto rawObservation = MakeScopeGuard([this, &observedRawCapacity, &rawWasObserved]() noexcept { + if(rawWasObserved) + { + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::RawReleased, observedRawCapacity, 0); + } + }); +#endif std::vector stored(static_cast(storedSize)); +#if SIMPLNX_BUILD_TESTS + observedRawCapacity = stored.capacity(); + rawWasObserved = true; + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::RawAcquired, observedRawCapacity, 0); +#endif #ifdef _WIN32 // Windows HDF5 owns the file range lock. Read raw bytes through HDF5, then // release Support::ApiLock() before worker inflation. @@ -263,11 +587,20 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk } uint32 ignoredReadFilterMask = 0; { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadAttempt, storedSize, 0); +#endif if(H5Dread_chunk(m_DatasetId, H5P_DEFAULT, offset.data(), &ignoredReadFilterMask, stored.data()) < 0) { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadFailed, storedSize, 0); +#endif throw std::runtime_error(fmt::format("ParallelChunkCodec: H5Dread_chunk failed on chunk {} of '{}:{}'", flatChunkIndex, m_FilePath.string(), m_DatasetPath)); } +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredRead, storedSize, 0); +#endif } // Keep the H5Dget_chunk_info_by_coord metadata snapshot. On Windows, H5Dread_chunk // can report a stale filter mask after a live writable chunk is rewritten. @@ -277,7 +610,25 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk // a short read from a recently extended file. auto readStoredBytes = [&]() -> std::ptrdiff_t { const nx::core::detail::FileHandle rawHandle = getPositionalReadHandle(); - return nx::core::detail::positionalRead(rawHandle, stored.data(), static_cast(storedSize), static_cast(storedAddress)); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadAttempt, storedSize, 0); +#endif + const auto result = nx::core::detail::positionalRead(rawHandle, stored.data(), static_cast(storedSize), static_cast(storedAddress)); +#if SIMPLNX_BUILD_TESTS + if(result == static_cast(storedSize)) + { + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredRead, storedSize, 0); + } + else if(result >= 0) + { + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadShort, static_cast(result), storedSize); + } + else + { + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredReadFailed, storedSize, 0); + } +#endif + return result; }; std::ptrdiff_t got = readStoredBytes(); @@ -290,7 +641,7 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk if(got != static_cast(storedSize)) { { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); if(H5Fflush(m_DatasetId, H5F_SCOPE_LOCAL) < 0) { throw std::runtime_error( @@ -308,7 +659,13 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk // Inflate or copy into the full padded chunk buffer. const usize nominalBytes = m_NominalChunkElements * m_ElementSize; +#if SIMPLNX_BUILD_TESTS + LegacyPayloadTicketForTesting nominalObservation; +#endif std::vector nominal(nominalBytes); +#if SIMPLNX_BUILD_TESTS + nominalObservation.acquire(m_FilePath, m_DatasetPath, nominal, LegacyPayloadKindForTesting::Nominal); +#endif // Bit zero marks skipped deflate. Such a chunk stores raw nominal bytes. const bool deflateSkipped = (filterMask & 0x1u) != 0u; @@ -319,6 +676,9 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk throw std::runtime_error(fmt::format("ParallelChunkCodec: uncompressed chunk {} size {} != nominal {} in '{}'", flatChunkIndex, storedSize, nominalBytes, m_FilePath.string())); } std::memcpy(nominal.data(), stored.data(), nominalBytes); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::SkippedDeflate, storedSize, nominalBytes); +#endif } else { @@ -329,12 +689,21 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk fmt::format("ParallelChunkCodec: nominal chunk {} size {} exceeds zlib's {}-byte limit in '{}'", flatChunkIndex, nominalBytes, std::numeric_limits::max(), m_FilePath.string())); } uLongf destLen = static_cast(nominalBytes); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::InflateAttempt, storedSize, nominalBytes); +#endif const int zret = uncompress(reinterpret_cast(nominal.data()), &destLen, reinterpret_cast(stored.data()), static_cast(storedSize)); if(zret != Z_OK || destLen != static_cast(nominalBytes)) { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::InflateFailed, storedSize, nominalBytes); +#endif throw std::runtime_error( fmt::format("ParallelChunkCodec: zlib uncompress failed (ret={}, got {} of {} bytes) on chunk {} of '{}'", zret, destLen, nominalBytes, flatChunkIndex, m_FilePath.string())); } +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::Inflated, storedSize, static_cast(destLen)); +#endif } // Interior chunks already match the clamped layout. Edge chunks extract the // in-bounds region for serial H5Dread parity. @@ -350,18 +719,30 @@ std::vector ParallelChunkCodec::inflateChunkFromInfo(uint64 flatChunk } if(isInterior) { +#if SIMPLNX_BUILD_TESTS + nominalObservation.forwardReturn(); +#endif return nominal; } const usize clampedTuples = product(clampedTupleDims); const usize compBytes = m_NumComponents * m_ElementSize; +#if SIMPLNX_BUILD_TESTS + LegacyPayloadTicketForTesting clampedObservation; +#endif std::vector clamped(clampedTuples * compBytes); +#if SIMPLNX_BUILD_TESTS + clampedObservation.acquire(m_FilePath, m_DatasetPath, clamped, LegacyPayloadKindForTesting::Edge); +#endif for(usize flatClamped = 0; flatClamped < clampedTuples; ++flatClamped) { const std::vector nd = flatToNd(static_cast(flatClamped), clampedTupleDims); const usize nominalTupleFlat = static_cast(ndToFlat(nd, m_ChunkShape)); std::memcpy(clamped.data() + flatClamped * compBytes, nominal.data() + nominalTupleFlat * compBytes, compBytes); } +#if SIMPLNX_BUILD_TESTS + clampedObservation.forwardReturn(); +#endif return clamped; } @@ -467,12 +848,18 @@ void ParallelChunkCodec::inflateChunksIntoSpan(nonstd::span out, nons } { - std::lock_guard hdf5Lock(Support::ApiLock()); - const hid_t fileSpace = H5Dget_space(m_DatasetId); - const hid_t memSpace = H5Screate_simple(static_cast(fullRank), memDims.data(), nullptr); - const hid_t dtype = H5Dget_type(m_DatasetId); - const herr_t selStatus = (fileSpace < 0) ? static_cast(-1) : H5Sselect_hyperslab(fileSpace, H5S_SELECT_SET, fileStart.data(), nullptr, count.data(), nullptr); - const herr_t readStatus = (fileSpace < 0 || memSpace < 0 || dtype < 0 || selStatus < 0) ? static_cast(-1) : H5Dread(m_DatasetId, dtype, memSpace, fileSpace, H5P_DEFAULT, clamped.data()); +#if SIMPLNX_BUILD_TESTS + const MetadataIdentityForTesting metadataIdentity(m_FilePath, m_DatasetPath); +#endif + const std::lock_guard hdf5Lock(Support::ApiLock()); + const hid_t fileSpace = SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Dget_space(m_DatasetId)); + const hid_t memSpace = SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Screate_simple(static_cast(fullRank), memDims.data(), nullptr)); + const hid_t dtype = SIMPLNX_OBSERVE_METADATA_CALL(MetadataTypeCall, H5Dget_type(m_DatasetId)); + const herr_t selStatus = (fileSpace < 0) ? static_cast(-1) : + SIMPLNX_OBSERVE_METADATA_CALL(MetadataSpaceCall, H5Sselect_hyperslab(fileSpace, H5S_SELECT_SET, fileStart.data(), nullptr, count.data(), nullptr)); + const herr_t readStatus = (fileSpace < 0 || memSpace < 0 || dtype < 0 || selStatus < 0) ? + static_cast(-1) : + SIMPLNX_OBSERVE_TYPED_READ(m_FilePath, m_DatasetPath, clamped.size(), H5Dread(m_DatasetId, dtype, memSpace, fileSpace, H5P_DEFAULT, clamped.data())); if(dtype >= 0) { H5Tclose(dtype); @@ -520,6 +907,9 @@ void ParallelChunkCodec::inflateChunksIntoSpan(nonstd::span out, nons auto loader = [&](uint64 idx) -> std::vector { const Extent bounds = getChunkBounds(idx, m_TupleShape, m_ChunkShape); const ChunkInfo info = peekChunkInfo(m_DatasetId, bounds, m_ComponentShape.size()); +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::ChunkRecordQuery, 1, 0); +#endif if(info.allocated) { return inflateChunkFromInfo(idx, bounds, info.storedAddress, info.storedSize, info.filterMask); @@ -660,9 +1050,12 @@ bool ParallelChunkCodec::writeCompressedChunkImpl(uint64 flatChunkIndex, nonstd: // Bit zero marks raw storage. Support::ApiLock() guards only the leaf write. { - std::lock_guard hdf5Lock(Support::ApiLock()); + const std::lock_guard hdf5Lock(Support::ApiLock()); if(H5Dwrite_chunk(m_DatasetId, H5P_DEFAULT, filterMask, offset.data(), storedBytes.size(), storedBytes.data()) >= 0) { +#if SIMPLNX_BUILD_TESTS + ObserveCodecIoForTesting(m_FilePath, m_DatasetPath, CodecIoEventForTesting::StoredWrite, storedBytes.size(), 0); +#endif return true; } } @@ -1048,3 +1441,16 @@ bool ParallelChunkCodec::deflateSpanIntoChunks(nonstd::span sou } } // namespace nx::core::HDF5 + +#if SIMPLNX_BUILD_TESTS +namespace nx::core::BoundedRead +{ +void ObserveTerminalForTesting(const std::filesystem::path* file, std::string_view dataset, bool requested, uint64 first, uint64 second) noexcept +{ + if(file != nullptr) + { + HDF5::ObserveCodecIoForTesting(*file, dataset, requested ? HDF5::CodecIoEventForTesting::DiagnosticRequested : HDF5::CodecIoEventForTesting::DiagnosticRetained, first, second); + } +} +} // namespace nx::core::BoundedRead +#endif diff --git a/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp b/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp index 7ef7ce99bc..4fddeefdee 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp @@ -1,6 +1,6 @@ #pragma once -#include "simplnx/Common/Extent.hpp" +#include "simplnx/Common/BoundedRead.hpp" #include "simplnx/Common/Types.hpp" #include "simplnx/Utilities/PositionalFileIO.hpp" #include "simplnx/simplnx_export.hpp" @@ -10,13 +10,473 @@ #include #include +#include #include +#include #include #include +#include #include namespace nx::core::HDF5 { +#if SIMPLNX_BUILD_TESTS +/** + * @enum CodecIoEventForTesting + * @brief Identifies measured codec work without retaining event records. + */ +enum class CodecIoEventForTesting +{ + StoredRead, ///< Complete successful raw read; input is transferred bytes. + Inflated, ///< Successful zlib decode; input is supplied buffer bytes, output is decoded bytes. + StoredWrite, ///< Successful raw write; input is stored bytes. + StoredReadAttempt, ///< Native read attempt; input is requested bytes. + StoredReadShort, ///< Short read; input is received bytes and output is requested bytes. + StoredReadFailed, ///< Failed native read; input is requested bytes. + InflateAttempt, ///< Decode attempt; input is stored bytes and output is expected bytes. + InflateFailed, ///< Failed decode; input is stored bytes and output is expected bytes. + SkippedDeflate, ///< Native bytes bypass zlib; input is stored bytes and output is nominal bytes. + ChunkRecordQuery, ///< One completed chunk-record query, including a failed native status. + RawAcquired, ///< Raw carrier acquired; input is actual capacity in bytes. + RawReleased, ///< Raw carrier destroyed; input is its prior capacity in bytes. + LegacyAcquired, ///< Payload acquired; input is capacity and output is its category. + LegacyReleased, ///< Payload destroyed; input is capacity and output is its category. + LegacyToReturned, ///< Category transfer; input is capacity and output is the previous category. + LegacyTransferred, ///< Cache takes ownership; input is capacity and output is its category. + TypedReadAttempt, ///< Typed H5Dread attempt; input is logical requested bytes. + TypedReadCompleted, ///< Successful typed read; input is logical bytes, not device traffic. + TypedReadFailed, ///< Failed typed H5Dread; input is requested bytes. + ContiguousReadAttempt, ///< Positional contiguous read attempt; input is requested bytes. + ContiguousReadCompleted, ///< Successful positional contiguous read; input is received bytes. + ContiguousReadFailed, ///< Failed or short contiguous read; input is requested bytes. + MetadataTypeCall, ///< One instrumented datatype API call. + MetadataLayoutCall, ///< One instrumented layout API call. + MetadataFilterCall, ///< One instrumented filter-list API call. + MetadataFillCall, ///< One instrumented fill-policy API call. + MetadataSpaceCall, ///< One instrumented dataspace or selection API call. + ExtentAcquired, ///< Returned-extent payload acquired; input is retained byte capacity. + ExtentReleased, ///< Failed extent payload destroyed; input is retained capacity. + ExtentReturned, ///< Successful extent return; input is capacity and output is logical bytes. + SamplerAcquired, ///< Sample carrier acquired; input is bytes and output is category. + SamplerReleased, ///< Sample carrier destroyed; input is bytes and output is category. + SamplerPublished, ///< Sample payload becomes output; input is bytes and output is category. + SamplerHistogramReturned, ///< Histogram helper returns charged payload; input is bytes. + SamplerHistogramAdopted, ///< Accumulator replacement; input is new capacity, output is destroyed capacity. + SamplerTypedAdopted, ///< Caller adopts the already-observed returned extent capacity. + SamplerReadRequest, ///< Bulk request; input is values and output is logical bytes. + SamplerReadCompleted, ///< Successful bounded read; input is values, output is logical bytes. + SamplerReserveRequest, ///< Allocation request; input is requested bytes and output is category. + SamplerWorkingReservation, ///< Shared reservation outside backend locks; input is grant bytes, output is conservative fixed sampler bytes. + SamplerInvalid, ///< Measurement metadata cannot be represented; production flow is unchanged. + PendingWriteScan, ///< Visited resident/parked entries during an optional clean-state proof. + DiagnosticRequested, ///< Terminal reserve request; input bytes, output zero vector or one string. + DiagnosticRetained, ///< Terminal retained record-vector bytes and message bytes; no pending ownership record. + BoundedDeferredRequest, ///< Recorded exact request delivered after identity protection; not a live event. + BoundedDeferredPublishedCapacity, ///< Recorded retained capacity already published; not a live event. + BoundedAdmissionCharge, ///< Conservative pre-service charge; input is bound bytes, output is carrier. + BoundedRequested, ///< Before owned storage service; input is bytes, output is carrier category. + BoundedAcquired, ///< Actual owned bytes acquired; input is bytes, output is carrier category. + BoundedPublished, ///< Owned scratch becomes retained store state; input is bytes, output is carrier. + BoundedReleased, ///< Actual owned bytes released; input is bytes, output is carrier category. + TypedWriteAttempt, ///< Typed H5Dwrite attempt; input is selected logical bytes. + TypedWriteCompleted, ///< Successful typed H5Dwrite; input is logical bytes, not stored or device bytes. + TypedWriteFailed, ///< Failed or throwing typed write; input is selected logical bytes. + TypedWriteSizeInvalid, ///< Unrepresentable logical byte count; invalidates measurement without changing the native call. + Count ///< Supplies the fixed observation-array size. +}; + +/** @brief Names narrow bounded-read failure sites on the existing quiescent test observer. */ +enum class BoundedFaultPointForTesting +{ + CheckedMetadataUnsupportedBuild, + NativeSizeQuery, + MetadataQuery, + ContradictoryRecord, + SecondPassUnpreparedDecoder, + SecondPassLargerRecord, + StoredAllocation, + NominalAllocation, + DecoderAllocation, + DecoderQuotaAfterPayload, + RawRead +}; + +/** + * @struct CodecIoObserverForTesting + * @brief Borrows a nonallocating callback while all selected codec work runs. + * + * Install and restore only while codec workers are quiescent. The context and borrowed identities outlive every callback. + */ +struct SIMPLNX_EXPORT CodecIoObserverForTesting +{ + void* context = nullptr; + void (*callback)(void*, const std::filesystem::path&, std::string_view, CodecIoEventForTesting, uint64 inputBytes, uint64 outputBytes) noexcept = nullptr; + bool (*boundedFault)(void*, const std::filesystem::path&, std::string_view, BoundedFaultPointForTesting) noexcept = nullptr; +}; + +/** @brief Registers synchronous bounded work with the existing observer quiescence counter. */ +class SIMPLNX_EXPORT BoundedReadTaskScopeForTesting +{ +public: + BoundedReadTaskScopeForTesting() noexcept; + ~BoundedReadTaskScopeForTesting() noexcept; + BoundedReadTaskScopeForTesting(const BoundedReadTaskScopeForTesting&) = delete; + BoundedReadTaskScopeForTesting(BoundedReadTaskScopeForTesting&&) = delete; + BoundedReadTaskScopeForTesting& operator=(const BoundedReadTaskScopeForTesting&) = delete; + BoundedReadTaskScopeForTesting& operator=(BoundedReadTaskScopeForTesting&&) = delete; + +private: + bool m_Active = false; +}; + +/** + * @brief Installs a test observer without allocating or locking. + * @param observer Supplies the callback and its borrowed context. + * @return Previous observer for a scoped restoration guard. + * @pre All codec workers are quiescent during replacement. + * + * Observed live payloads or tasks refuse replacement and record integrity failure. This check does not join or synchronize workers. + */ +SIMPLNX_EXPORT CodecIoObserverForTesting SetCodecIoObserverForTesting(CodecIoObserverForTesting observer) noexcept; + +/** + * @brief Invokes an optional fault leaf through the shared scoped observer. + * @param file Borrows the backend file identity. + * @param dataset Borrows the dataset identity. + * @param point Selects the narrow failure boundary. + * @return True when this leaf must inject the selected failure. + */ +SIMPLNX_EXPORT bool InjectCodecFaultForTesting(const std::filesystem::path& file, std::string_view dataset, BoundedFaultPointForTesting point) noexcept; + +/** + * @brief Emits one observation to the installed test callback. + * @param filePath Borrows the source identity for this synchronous callback. + * @param datasetPath Borrows the dataset identity. + * @param event Identifies the measured operation. + * @param inputBytes Supplies the event's input count. + * @param outputBytes Supplies the event's output count. + * @pre The callback does not allocate, lock, throw, or reenter storage code. + */ +SIMPLNX_EXPORT void ObserveCodecIoForTesting(const std::filesystem::path& filePath, std::string_view datasetPath, CodecIoEventForTesting event, uint64 inputBytes, uint64 outputBytes) noexcept; + +/** + * @enum LegacyPayloadKindForTesting + * @brief Identifies measured legacy payload storage across moves. + */ +enum class LegacyPayloadKindForTesting : uint8 +{ + Nominal, ///< Full decoded chunk image. + Edge, ///< Clamped edge image overlapping the nominal image. + Returned ///< Image returned to a known loader or consumer. +}; + +/** + * @namespace testing_detail + * @brief Contains fixed records for the scoped ownership observation. + */ +namespace testing_detail +{ +/** + * @struct LegacyPayloadRecordForTesting + * @brief Keeps one nonowning receipt without allocating a registry. + */ +struct LegacyPayloadRecordForTesting +{ + const std::filesystem::path* file = nullptr; + std::string_view dataset; + const std::byte* data = nullptr; + usize capacity = 0; + LegacyPayloadKindForTesting kind = LegacyPayloadKindForTesting::Nominal; + bool charged = false; +}; +} // namespace testing_detail + +/** + * @struct CodecOwnershipStateForTesting + * @brief Reports joined ownership integrity without reading other workers' TLS. + */ +struct CodecOwnershipStateForTesting +{ + uint64 outstandingBuffers = 0; + uint64 activeTasks = 0; + uint64 integrityFailures = 0; + uint64 unobservedReturns = 0; + uint64 errorAggregations = 0; +}; + +/** + * @brief Reads bounded atomic ownership counters. + * @return Snapshot; compare integrity and unobserved counters with their entry values. + */ +SIMPLNX_EXPORT CodecOwnershipStateForTesting GetCodecOwnershipStateForTesting() noexcept; + +/** + * @class LegacyPayloadTicketForTesting + * @brief Observes one byte vector until destruction or cache ownership transfer. + * + * Declare this ticket before its vector. Moves transfer observation, never vector ownership. Tickets stay on their acquisition thread. + */ +class SIMPLNX_EXPORT LegacyPayloadTicketForTesting +{ +public: + /** + * @brief Creates an uncharged ticket without allocating storage. + */ + LegacyPayloadTicketForTesting() = default; + /** + * @brief Releases the observation after its associated vector dies. + */ + ~LegacyPayloadTicketForTesting() noexcept; + LegacyPayloadTicketForTesting(const LegacyPayloadTicketForTesting&) = delete; + LegacyPayloadTicketForTesting& operator=(const LegacyPayloadTicketForTesting&) = delete; + /** + * @brief Moves the fixed receipt without changing charged bytes. + * @param other Supplies the receipt and becomes empty. + */ + LegacyPayloadTicketForTesting(LegacyPayloadTicketForTesting&& other) noexcept; + LegacyPayloadTicketForTesting& operator=(LegacyPayloadTicketForTesting&&) = delete; + + /** + * @brief Starts observation after a successful payload allocation. + * @param file Borrows the stable file identity. + * @param dataset Borrows the stable dataset identity. + * @param bytes Supplies the allocated byte vector. + * @param kind Identifies its initial storage category. + * @pre This ticket is empty and the identities outlive observation. + */ + void acquire(const std::filesystem::path& file, std::string_view dataset, const std::vector& bytes, LegacyPayloadKindForTesting kind) noexcept; + /** + * @brief Consumes the immediate vector-return receipt without allocating. + * @param bytes Supplies the returned vector before any fallible caller work. + * @param allowUnobserved Permits explicitly unobserved sparse or parked-source returns. + * @pre This ticket is empty; the corresponding vector-return expression just completed. + */ + void consumeReturn(const std::vector& bytes, bool allowUnobserved = false) noexcept; + /** + * @brief Consumes a replacement receipt before releasing the destroyed old payload's observation. + * @param bytes Supplies the newly assigned vector. + * @param allowUnobserved Permits a sparse or parked-source replacement without a codec receipt. + * @pre The vector assignment just destroyed the old payload; no callback or fallible work has followed it. + */ + void replaceDestroyedFromReturn(const std::vector& bytes, bool allowUnobserved = false) noexcept; + /** + * @brief Publishes one receipt for an immediately following nonthrowing vector return. + * @pre The thread's pending slot is empty and no fallible work follows before return. + */ + void forwardReturn() noexcept; + /** + * @brief Releases observation after destruction of a replaced vector buffer. + * @pre The observed payload has already been destroyed; no later release belongs to this ticket. + */ + void releaseDestroyed() noexcept; + /** + * @brief Ends operation-scratch observation at the actual cache move or swap. + * @pre Cache-owned storage has just taken the buffer matched before mutation. + */ + void transferToCache() noexcept; + /** + * @brief Checks a candidate handoff without changing observation. + * @param bytes Supplies the source vector before a move or swap. + * @return True only when the charged pointer and capacity match. + */ + bool matches(const std::vector& bytes) const noexcept; + bool charged() const noexcept; + +private: + testing_detail::LegacyPayloadRecordForTesting m_Record; +}; + +/** + * @class LegacyPayloadTaskScopeForTesting + * @brief Checks the current worker's pending receipt after its local results die. + */ +class SIMPLNX_EXPORT LegacyPayloadTaskScopeForTesting +{ +public: + /** + * @brief Starts a task observation while a shared observer is installed. + */ + LegacyPayloadTaskScopeForTesting() noexcept; + /** + * @brief Records and clears a stranded receipt without throwing from cleanup. + */ + ~LegacyPayloadTaskScopeForTesting() noexcept; + /** + * @brief Checks released inner payload state while the worker's error mutex is held. + * @pre Local result and ticket scopes have already unwound. + */ + void observeExceptionAggregation() noexcept; + LegacyPayloadTaskScopeForTesting(const LegacyPayloadTaskScopeForTesting&) = delete; + LegacyPayloadTaskScopeForTesting(LegacyPayloadTaskScopeForTesting&&) = delete; + LegacyPayloadTaskScopeForTesting& operator=(const LegacyPayloadTaskScopeForTesting&) = delete; + LegacyPayloadTaskScopeForTesting& operator=(LegacyPayloadTaskScopeForTesting&&) = delete; + +private: + bool m_Active = false; + uint64 m_ThreadOutstandingAtEntry = 0; + LegacyPayloadTicketForTesting* m_AdmissionAtEntry = nullptr; +}; + +/** + * @class LegacyPayloadAdmissionScopeForTesting + * @brief Borrows one worker-owned ticket across synchronous clean-cache admission. + */ +class SIMPLNX_EXPORT LegacyPayloadAdmissionScopeForTesting +{ +public: + /** + * @brief Selects the ticket inspected by actual cache move/swap sites. + * @param ticket Borrows the candidate payload ticket through the synchronous call. + */ + explicit LegacyPayloadAdmissionScopeForTesting(LegacyPayloadTicketForTesting& ticket) noexcept; + /** + * @brief Restores the previous thread-local admission pointer. + */ + ~LegacyPayloadAdmissionScopeForTesting() noexcept; + LegacyPayloadAdmissionScopeForTesting(const LegacyPayloadAdmissionScopeForTesting&) = delete; + LegacyPayloadAdmissionScopeForTesting(LegacyPayloadAdmissionScopeForTesting&&) = delete; + LegacyPayloadAdmissionScopeForTesting& operator=(const LegacyPayloadAdmissionScopeForTesting&) = delete; + LegacyPayloadAdmissionScopeForTesting& operator=(LegacyPayloadAdmissionScopeForTesting&&) = delete; + +private: + LegacyPayloadTicketForTesting* m_Previous = nullptr; +}; + +/** + * @brief Matches the borrowed admission ticket before a cache move/swap. + * @param bytes Supplies the still-unmodified caller payload. + * @return Matching ticket or null; empty preparation and unrelated bytes do not match. + */ +SIMPLNX_EXPORT LegacyPayloadTicketForTesting* MatchLegacyAdmissionForTesting(const std::vector& bytes) noexcept; + +/** + * @class MetadataIdentityForTesting + * @brief Borrows a read operation's identity for named native metadata calls. + */ +class SIMPLNX_EXPORT MetadataIdentityForTesting +{ +public: + /** + * @brief Selects a stable identity until this scope ends. + * @param file Borrows the attached file path. + * @param dataset Borrows the dataset name. + */ + MetadataIdentityForTesting(const std::filesystem::path& file, std::string_view dataset) noexcept; + /** + * @brief Restores the preceding thread-local identity. + */ + ~MetadataIdentityForTesting() noexcept; + MetadataIdentityForTesting(const MetadataIdentityForTesting&) = delete; + MetadataIdentityForTesting(MetadataIdentityForTesting&&) = delete; + MetadataIdentityForTesting& operator=(const MetadataIdentityForTesting&) = delete; + MetadataIdentityForTesting& operator=(MetadataIdentityForTesting&&) = delete; + +private: + const std::filesystem::path* m_PreviousFile = nullptr; + std::string_view m_PreviousDataset; +}; + +/** + * @brief Records one attempted metadata query under the current scoped identity. + * @param event Selects the named query family. + */ +SIMPLNX_EXPORT void ObserveMetadataCallForTesting(CodecIoEventForTesting event) noexcept; + +/** + * @brief Runs one native metadata expression exactly once, then records the call. + * @tparam Function Supplies the nonallocating native invocation. + * @param event Selects the query family. + * @param function Performs the existing native call. + * @return The unchanged native result, including failed status. + */ +template +auto InvokeMetadataForTesting(CodecIoEventForTesting event, Function&& function) -> decltype(function()) +{ + auto result = function(); + ObserveMetadataCallForTesting(event); + return result; +} + +/** + * @brief Runs one typed HDF5 read and reports actual attempt/completion boundaries. + * @tparam Function Supplies the existing H5Dread expression. + * @param file Borrows the stable file identity. + * @param dataset Borrows the dataset identity. + * @param bytes Supplies logical requested bytes, not compressed or device bytes. + * @param function Performs the native read exactly once. + * @return Unchanged native read status. + */ +template +auto InvokeTypedReadForTesting(const std::filesystem::path& file, std::string_view dataset, uint64 bytes, Function&& function) -> decltype(function()) +{ + ObserveCodecIoForTesting(file, dataset, CodecIoEventForTesting::TypedReadAttempt, bytes, 0); + auto result = function(); + ObserveCodecIoForTesting(file, dataset, result >= 0 ? CodecIoEventForTesting::TypedReadCompleted : CodecIoEventForTesting::TypedReadFailed, bytes, 0); + return result; +} + +/** + * @brief Checks the logical byte count for an observed typed-write selection. + * @param dimensions Selected scalar counts, including component dimensions. + * @param elementBytes Bytes in one native scalar. + * @return Logical bytes, or no value when the count cannot be represented. + */ +inline std::optional CheckedTypedWriteBytesForTesting(nonstd::span dimensions, uint64 elementBytes) noexcept +{ + if(elementBytes == 0) + { + return std::nullopt; + } + // A zero extent makes the selection empty even when an earlier product would overflow. + for(const hsize_t extent : dimensions) + { + if(extent == 0) + { + return uint64{0}; + } + } + uint64 bytes = elementBytes; + for(const hsize_t extent : dimensions) + { + if(bytes > std::numeric_limits::max() / extent) + { + return std::nullopt; + } + bytes *= extent; + } + return bytes; +} + +/** + * @brief Runs one typed write and preserves its return value or exception. + * @tparam Function Supplies the native write expression. + * @param file Borrows the selected file identity. + * @param dataset Borrows the selected dataset identity. + * @param bytes Supplies checked logical bytes; absence invalidates the measurement only. + * @param function Performs the unchanged native call exactly once. + * @return The unchanged native status. + */ +template +auto InvokeTypedWriteForTesting(const std::filesystem::path& file, std::string_view dataset, std::optional bytes, Function&& function) -> decltype(function()) +{ + if(!bytes) + { + ObserveCodecIoForTesting(file, dataset, CodecIoEventForTesting::TypedWriteSizeInvalid, 0, 0); + return function(); + } + ObserveCodecIoForTesting(file, dataset, CodecIoEventForTesting::TypedWriteAttempt, *bytes, 0); + try + { + auto result = function(); + ObserveCodecIoForTesting(file, dataset, result >= 0 ? CodecIoEventForTesting::TypedWriteCompleted : CodecIoEventForTesting::TypedWriteFailed, *bytes, 0); + return result; + } catch(...) + { + ObserveCodecIoForTesting(file, dataset, CodecIoEventForTesting::TypedWriteFailed, *bytes, 0); + throw; + } +} +#endif /** * @namespace nx::core::HDF5 @@ -123,6 +583,47 @@ class SIMPLNX_EXPORT ParallelChunkCodec * @return True for one deflate filter, exact file/memory representation, and compatible byte order. */ bool isEligible() const; +#if SIMPLNX_BUILD_TESTS + /** + * @brief Reports observable retained native/string/shape capacities. + * @return Native-string bytes including terminator, dataset bytes including terminator, and three vector payloads. + * @note GNU path component-list storage is admitted separately and is not observed by this receipt. + */ + uint64 boundedMetadataCapacityBytesForTesting() const noexcept; +#endif + + /** + * @brief Reads checked metadata for one full-rank physical chunk origin. + * @param fullRankChunkOffset Supplies tuple and component coordinates. + * @return A sparse/allocated record, or an invalid query/validation result. + * @note An unreviewed diagnostic build returns invalid with empty element collections + * before metadata work. Supported builds retain reason-coded errors. The caller + * funds BoundedRead::EmptyInvalidResultControlBytes() for mandatory + * typed return/move controls; debug empty-container bookkeeping remains permitted. + */ + Result getChunkInfoChecked(nonstd::span fullRankChunkOffset) const; + + /** + * @brief Reads full-value coordinates without exceeding admitted backend scratch. + * @param fullValueExtent Selects tuple and component axes in row-major order. + * @param destination Receives the exact selected scalar byte count. + * @param scratchBudgetBytes Funds metadata, raw/decoded carriers, zlib and diagnostics. + * @return True when complete; false before payload transfer; invalid on a real failure. + * @note The caller funds mandatory empty Result controls outside scratchBudgetBytes. + * Subminimum calls start no preparation, payload or diagnostic-text allocation. + * Iterator-debug empty-vector bookkeeping retains its normal ABI behavior. + */ + Result readExtentIntoBufferBounded(const Extent& fullValueExtent, nonstd::span destination, uint64 scratchBudgetBytes) const; + + /** + * @brief Uses the outer store's diagnostic owner after its minimum is funded. + * @param fullValueExtent Selects all value-space dimensions. + * @param destination Receives selected scalar bytes. + * @param payloadBudgetBytes Excludes the outer diagnostic allowance. + * @param diagnostic Borrows the sole fixed diagnostic carrier. + * @return Fixed state; payload scratch has been destroyed before return. + */ + BoundedRead::Status readExtentIntoBufferBoundedState(const Extent& fullValueExtent, nonstd::span destination, uint64 payloadBudgetBytes, BoundedRead::Diagnostic& diagnostic) const; /** * @brief Inflates one chunk into an edge-clamped byte buffer. @@ -376,6 +877,9 @@ class SIMPLNX_EXPORT ParallelChunkCodec int32 m_DeflateLevel = 1; // Eligibility-probe deflate level for compress2. uint64 m_NumChunks = 1; // Total logical chunks for index validation. bool m_Eligible = false; + // Stable predefined native identifier; never retains the constructor's borrowed type. + hid_t m_BoundedNativeType = H5I_INVALID_HID; + bool m_BoundedTypeQueryFailed = false; #ifndef _WIN32 // Serializes short-read recovery only. Successful positional reads remain lock free. @@ -386,3 +890,43 @@ class SIMPLNX_EXPORT ParallelChunkCodec }; } // namespace nx::core::HDF5 + +/** + * @def SIMPLNX_OBSERVE_METADATA_CALL + * @brief Preserves one native expression and observes it only in test builds. + * @param event Names the metadata call family. + * @param expression Supplies the native expression, evaluated exactly once. + */ +#if SIMPLNX_BUILD_TESTS +#define SIMPLNX_OBSERVE_METADATA_CALL(event, expression) nx::core::HDF5::InvokeMetadataForTesting(nx::core::HDF5::CodecIoEventForTesting::event, [&]() { return (expression); }) +#else +#define SIMPLNX_OBSERVE_METADATA_CALL(event, expression) (expression) +#endif + +/** + * @def SIMPLNX_OBSERVE_TYPED_READ + * @brief Preserves a typed read and observes its native boundary only in test builds. + * @param file Borrows the file identity. + * @param dataset Borrows the dataset identity. + * @param bytes Supplies logical requested bytes. + * @param expression Supplies H5Dread, evaluated exactly once. + */ +#if SIMPLNX_BUILD_TESTS +#define SIMPLNX_OBSERVE_TYPED_READ(file, dataset, bytes, expression) nx::core::HDF5::InvokeTypedReadForTesting(file, dataset, bytes, [&]() { return (expression); }) +#else +#define SIMPLNX_OBSERVE_TYPED_READ(file, dataset, bytes, expression) (expression) +#endif + +/** + * @def SIMPLNX_OBSERVE_TYPED_WRITE + * @brief Preserves a typed write and observes its native boundary only in test builds. + * @param file Borrows the file identity. + * @param dataset Borrows the dataset identity. + * @param bytes Supplies checked logical bytes. + * @param expression Supplies H5Dwrite, evaluated exactly once. + */ +#if SIMPLNX_BUILD_TESTS +#define SIMPLNX_OBSERVE_TYPED_WRITE(file, dataset, bytes, expression) nx::core::HDF5::InvokeTypedWriteForTesting(file, dataset, bytes, [&]() { return (expression); }) +#else +#define SIMPLNX_OBSERVE_TYPED_WRITE(file, dataset, bytes, expression) (expression) +#endif diff --git a/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkLoop.hpp b/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkLoop.hpp index 131d350aa2..92ceaf6901 100644 --- a/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkLoop.hpp +++ b/src/simplnx/Utilities/Parsing/HDF5/ParallelChunkLoop.hpp @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include namespace nx::core::HDF5 { @@ -100,9 +102,21 @@ void ParallelLoadChunks(nonstd::span chunkIndices, const std::func std::exception_ptr firstError = nullptr; const auto loadOne = [&](usize i) { +#if SIMPLNX_BUILD_TESTS + LegacyPayloadTaskScopeForTesting ownershipTask; +#endif try { +#if SIMPLNX_BUILD_TESTS + LegacyPayloadTicketForTesting returnedObservation; +#endif Result result = loader(chunkIndices[i]); +#if SIMPLNX_BUILD_TESTS + if constexpr(std::is_same_v>) + { + returnedObservation.consumeReturn(result, true); + } +#endif // A sink can write one pre-sized local slot without sharing that slot. sink(i, std::move(result)); } catch(const UnallocatedChunkError&) @@ -114,6 +128,9 @@ void ParallelLoadChunks(nonstd::span chunkIndices, const std::func { // Record the first observed failure. Other scheduled positions still run. std::lock_guard lk(errMutex); +#if SIMPLNX_BUILD_TESTS + ownershipTask.observeExceptionAggregation(); +#endif if(!firstError) { firstError = std::current_exception(); diff --git a/src/simplnx/Utilities/StoreCopyUtilities.cpp b/src/simplnx/Utilities/StoreCopyUtilities.cpp index cc99d4ed51..4100d95829 100644 --- a/src/simplnx/Utilities/StoreCopyUtilities.cpp +++ b/src/simplnx/Utilities/StoreCopyUtilities.cpp @@ -131,6 +131,7 @@ struct CopyNumericFunctor { throw std::runtime_error(result.errors().front().message); } + result.value()->setInitValue(typedSource->getInitValue()); return std::move(result.value()); } auto destination = manager->dataStoreCreationFnc(format)(source.getDataType(), source.getTupleShape(), source.getComponentShape(), std::nullopt, DataStoreInitializationMode::Default); @@ -162,6 +163,7 @@ struct CopyNumericFunctor throw std::runtime_error(message); } } + typedDestination->setInitValue(typedSource->getInitValue()); return destination; } }; diff --git a/src/simplnx/Utilities/StoreCopyUtilities.hpp b/src/simplnx/Utilities/StoreCopyUtilities.hpp index 4de8c3d4c1..c7f465df2b 100644 --- a/src/simplnx/Utilities/StoreCopyUtilities.hpp +++ b/src/simplnx/Utilities/StoreCopyUtilities.hpp @@ -53,7 +53,7 @@ SIMPLNX_EXPORT uint64 CalculateStoreCopyBytes(const ShapeType& tupleShape, const * @brief Copies numeric values or placeholder metadata into a selected storage format. * @param source Store to copy without modification. * @param destinationFormat Resolved destination format; empty selects in-memory storage. - * @return Independent owned store with the same value type and shapes. + * @return Independent owned store with the same value type, shapes and growth initializer. * @throws std::runtime_error If format selection, factory validation, or transfer fails. * @throws std::bad_alloc If an allocation fails. * diff --git a/test/ArraySelectionParameterTest.cpp b/test/ArraySelectionParameterTest.cpp new file mode 100644 index 0000000000..29ed230a5c --- /dev/null +++ b/test/ArraySelectionParameterTest.cpp @@ -0,0 +1,180 @@ +#include "simplnx/Parameters/ArraySelectionParameter.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/EmptyDataStore.hpp" +#include "simplnx/DataStructure/StringArray.hpp" + +#include + +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +/** + * @class ReportingOocStore + * @brief Reports OOC storage while values remain in memory. + * + * Location validation reads only store metadata, so this fixture does not need an OOC backend. + */ +class ReportingOocStore final : public DataStore +{ +public: + using DataStore::DataStore; + + ~ReportingOocStore() override = default; + ReportingOocStore(const ReportingOocStore&) = default; + ReportingOocStore(ReportingOocStore&&) = default; + ReportingOocStore& operator=(const ReportingOocStore&) = delete; + ReportingOocStore& operator=(ReportingOocStore&&) = default; + + /** + * @brief Reports OOC storage for location validation. + * @return OutOfCore for this test store. + */ + IDataStore::StoreType getStoreType() const override + { + return IDataStore::StoreType::OutOfCore; + } +}; + +/** + * @brief Checks location validation for a parameter and its clone. + * @param dataStructure Contains the selected array. + * @param path Identifies the selected array. + * @param location Specifies the required storage location. + * @param expectedValid Specifies the expected validation result. + */ +void checkLocation(const DataStructure& dataStructure, const DataPath& path, ArraySelectionParameter::DataLocation location, bool expectedValid) +{ + ArraySelectionParameter parameter("values", "Values", "Select values.", path, {DataType::int32}, {{1}}, location); + auto copy = parameter.clone(); + const auto* typedCopy = dynamic_cast(copy.get()); + REQUIRE(typedCopy != nullptr); + CHECK(parameter.allowedDataLocations() == location); + CHECK(typedCopy->allowedDataLocations() == location); + + const auto originalResult = parameter.validate(dataStructure, std::any{path}); + const auto copyResult = typedCopy->validate(dataStructure, std::any{path}); + CHECK(originalResult.valid() == expectedValid); + CHECK(copyResult.valid() == expectedValid); + + if(!expectedValid) + { + for(const auto* result : {&originalResult, ©Result}) + { + if(result->invalid()) + { + CHECK_FALSE(result->errors().empty()); + if(!result->errors().empty()) + { + CHECK(result->errors().front().code == FilterParameter::Constants::k_Validate_DataLocation_Error); + CHECK(result->errors().front().message.find(path.toString()) != std::string::npos); + CHECK(result->errors().front().message.find("in-memory") != std::string::npos); + CHECK(result->errors().front().message.find("out-of-core") != std::string::npos); + } + } + } + } +} +} // namespace + +TEST_CASE("ArraySelectionParameter location and clone", "[simplnx][Parameters]") +{ + DataStructure dataStructure; + const DataPath residentPath({"Resident"}); + const DataPath reportedOocPath({"ReportedOoc"}); + const DataPath emptyPath({"Empty"}); + + REQUIRE(DataArray::Create(dataStructure, residentPath.getTargetName(), std::make_shared>(ShapeType{2}, ShapeType{1}, 0)) != nullptr); + REQUIRE(DataArray::Create(dataStructure, reportedOocPath.getTargetName(), std::make_shared(ShapeType{2}, ShapeType{1}, 0)) != nullptr); + auto emptyStoreResult = EmptyDataStore::Create({2}, {1}, ""); + REQUIRE(emptyStoreResult.valid()); + std::shared_ptr> emptyStore(std::move(emptyStoreResult.value())); + REQUIRE(DataArray::Create(dataStructure, emptyPath.getTargetName(), std::move(emptyStore)) != nullptr); + + struct StoreCase + { + DataPath path; + IDataStore::StoreType actual; + }; + const std::vector stores = {{residentPath, IDataStore::StoreType::InMemory}, {reportedOocPath, IDataStore::StoreType::OutOfCore}, {emptyPath, IDataStore::StoreType::Empty}}; + + const std::vector locations = {ArraySelectionParameter::DataLocation::Any, ArraySelectionParameter::DataLocation::InMemory, + ArraySelectionParameter::DataLocation::OutOfCore}; + for(const auto& store : stores) + { + const auto* array = dataStructure.getDataAs(store.path); + REQUIRE(array != nullptr); + CHECK(array->getStoreType() == store.actual); + for(const auto location : locations) + { + INFO("array: " << store.path.toString()); + const bool expectedValid = location == ArraySelectionParameter::DataLocation::Any || store.actual == IDataStore::StoreType::Empty || + (location == ArraySelectionParameter::DataLocation::InMemory && store.actual == IDataStore::StoreType::InMemory) || + (location == ArraySelectionParameter::DataLocation::OutOfCore && store.actual == IDataStore::StoreType::OutOfCore); + checkLocation(dataStructure, store.path, location, expectedValid); + } + } +} + +TEST_CASE("ArraySelectionParameter keeps type and shape checks for empty stores", "[simplnx][Parameters]") +{ + DataStructure dataStructure; + auto wrongTypeStore = EmptyDataStore::Create({2}, {1}, ""); + REQUIRE(wrongTypeStore.valid()); + std::shared_ptr> floatStore(std::move(wrongTypeStore.value())); + REQUIRE(DataArray::Create(dataStructure, "WrongType", std::move(floatStore)) != nullptr); + + auto wrongShapeStore = EmptyDataStore::Create({2}, {2}, ""); + REQUIRE(wrongShapeStore.valid()); + std::shared_ptr> shapeStore(std::move(wrongShapeStore.value())); + REQUIRE(DataArray::Create(dataStructure, "WrongShape", std::move(shapeStore)) != nullptr); + + for(const auto location : {ArraySelectionParameter::DataLocation::Any, ArraySelectionParameter::DataLocation::InMemory, ArraySelectionParameter::DataLocation::OutOfCore}) + { + for(const auto& path : {DataPath({"WrongType"}), DataPath({"WrongShape"})}) + { + ArraySelectionParameter parameter("values", "Values", "Select values.", path, {DataType::int32}, {{1}}, location); + auto copy = parameter.clone(); + const auto* typedCopy = dynamic_cast(copy.get()); + REQUIRE(typedCopy != nullptr); + CHECK(typedCopy->allowedDataLocations() == location); + const auto originalResult = parameter.validate(dataStructure, std::any{path}); + const auto copyResult = typedCopy->validate(dataStructure, std::any{path}); + const auto expectedCode = path.getTargetName() == "WrongType" ? FilterParameter::Constants::k_Validate_AllowedType_Error : FilterParameter::Constants::k_Validate_TupleShapeValue; + for(const auto* result : {&originalResult, ©Result}) + { + CHECK(result->invalid()); + if(result->invalid()) + { + CHECK_FALSE(result->errors().empty()); + if(!result->errors().empty()) + { + CHECK(result->errors().front().code == expectedCode); + } + } + } + } + } +} + +TEST_CASE("ArraySelectionParameter retains StringArray and default location behavior", "[simplnx][Parameters]") +{ + DataStructure dataStructure; + const DataPath path({"Strings"}); + REQUIRE(StringArray::CreateWithValues(dataStructure, path.getTargetName(), {2}, {"one", "two"}) != nullptr); + + ArraySelectionParameter defaultParameter("values", "Values", "Select values.", path, {DataType::int32}, {{1}}); + CHECK(defaultParameter.allowedDataLocations() == ArraySelectionParameter::DataLocation::Any); + + for(const auto location : {ArraySelectionParameter::DataLocation::Any, ArraySelectionParameter::DataLocation::InMemory, ArraySelectionParameter::DataLocation::OutOfCore}) + { + checkLocation(dataStructure, path, location, true); + } +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3fc67be1c3..2f171303a5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,9 +28,12 @@ add_subdirectory(ConversionTest) set(SIMPLNX_TEST_SRCS simplnx_test_main.cpp + ProfileIsolationContract.cpp AppTest.cpp + ArraySelectionParameterTest.cpp ArgumentsTest.cpp BitTest.cpp + CheckedFlushTest.cpp ChunkIndexTest.cpp ChunkShapePolicyTest.cpp DataArrayTest.cpp @@ -51,6 +54,7 @@ set(SIMPLNX_TEST_SRCS ImageRotationUtilitiesTest.cpp FilePathGeneratorTest.cpp FilterTest.cpp + FloatArrayComparisonTest.cpp GeometryTest.cpp GeometryBulkContractsTest.cpp GeometryTestUtilities.hpp @@ -68,6 +72,7 @@ set(SIMPLNX_TEST_SRCS PluginTest.cpp ParametersTest.cpp PipelineFilterMessagingTest.cpp + PipelineFlushTest.cpp PipelineSaveTest.cpp PrebuiltPipelineTest.cpp UuidTest.cpp @@ -83,6 +88,8 @@ set(SIMPLNX_TEST_SRCS StorageFormatPlanTest.cpp DataStoreFormatResolverTest.cpp DataStorageModeMigrationTest.cpp + OocThresholdScopeTest.cpp + PreferencesSizeGBTest.cpp TiffImageIOTest.cpp ) @@ -136,12 +143,54 @@ target_compile_definitions(simplnx_test target_compile_options(simplnx_test PRIVATE $<$:/MP> + # Test templates can exceed MSVC's default object section limit. + $<$:/bigobj> ) target_include_directories(simplnx_test PRIVATE ${SIMPLNX_GENERATED_DIR}) catch_discover_tests(simplnx_test) +add_test(NAME "simplnx_test_profile_isolation" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -P "${CMAKE_CURRENT_SOURCE_DIR}/VerifyTestProfileIsolation.cmake") +set_tests_properties("simplnx_test_profile_isolation" PROPERTIES RUN_SERIAL TRUE TIMEOUT 150) + +add_executable(ProfileIsolationLifecycleProbe ProfileIsolationLifecycleProbe.cpp) +target_link_libraries(ProfileIsolationLifecycleProbe PRIVATE simplnx::TestProfile) +simplnx_enable_warnings(TARGET ProfileIsolationLifecycleProbe) +add_test(NAME "simplnx_test_profile_lifecycle" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_PROBE_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -P "${CMAKE_CURRENT_SOURCE_DIR}/VerifyTestProfileLifecycle.cmake") +set_tests_properties("simplnx_test_profile_lifecycle" PROPERTIES RUN_SERIAL TRUE TIMEOUT 150) + +add_executable(P1BenchmarkPreferenceSnapshotTest + ${simplnx_SOURCE_DIR}/src/Plugins/ImageProcessing/benchmark/BenchmarkPreferenceSnapshotTest.cpp) +target_link_libraries(P1BenchmarkPreferenceSnapshotTest PRIVATE simplnx::simplnx Catch2::Catch2) +simplnx_enable_warnings(TARGET P1BenchmarkPreferenceSnapshotTest) +add_test(NAME "P1 benchmark preference snapshot" + COMMAND P1BenchmarkPreferenceSnapshotTest "[Core][Preferences][OocThreshold]" --warn NoTests) +set_tests_properties("P1 benchmark preference snapshot" PROPERTIES RUN_SERIAL TRUE TIMEOUT 30) + +foreach(p1_case IN ITEMS + "P1 owned shared sentinel restores exact threshold and presence" + "P1 owned public algorithm scope retains maximum threshold" + "P1 owned Application retains startup correction through clean reload") + string(MAKE_C_IDENTIFIER "${p1_case}" p1_case_id) + add_test(NAME "P1_owned_${p1_case_id}" + COMMAND ${CMAKE_COMMAND} + "-DP1_TEST_EXECUTABLE=$" + "-DP1_TEST_NAME=${p1_case}" + "-DP1_ROOT_BASE=${CMAKE_CURRENT_BINARY_DIR}/P1Owned" + -P "${CMAKE_CURRENT_SOURCE_DIR}/OwnedP1CatchCase.cmake") + set_tests_properties("P1_owned_${p1_case_id}" PROPERTIES RUN_SERIAL TRUE TIMEOUT 150) +endforeach() + if(EXISTS "${DREAM3D_DATA_DIR}" AND SIMPLNX_DOWNLOAD_TEST_FILES) diff --git a/test/CacheMemoryBudgetManagerTest.cpp b/test/CacheMemoryBudgetManagerTest.cpp index 29948a9bf6..1932c6657f 100644 --- a/test/CacheMemoryBudgetManagerTest.cpp +++ b/test/CacheMemoryBudgetManagerTest.cpp @@ -776,3 +776,181 @@ TEST_CASE("CacheMemoryBudgetManager cache eviction accounts for active working m REQUIRE(mgr.reservedWorkingMemoryBytes() == 0); mgr.setBudgetBytes(CacheMemoryBudgetManager::defaultBudgetBytes()); } + +TEST_CASE("CacheMemoryBudgetManager exact working reservations require registered headroom and quota", "[CacheMemoryBudgetManager]") +{ + auto& mgr = CacheMemoryBudgetManager::instance(); + mgr.clear(); + mgr.setBudgetBytes(1000); + + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(0).sizeBytes() == 0); + auto [cacheHandle, evictions] = mgr.allocate("exact-working", "entry", 750, []() {}); + REQUIRE(evictions.empty()); + + auto exact = mgr.tryReserveWorkingMemoryWithoutEviction(250); + REQUIRE(exact.sizeBytes() == 250); + CHECK(mgr.usedBytes() == 750); + CHECK(mgr.reservedWorkingMemoryBytes() == 250); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(1).sizeBytes() == 0); + + exact.shrinkTo(100); + CHECK(mgr.reservedWorkingMemoryBytes() == 100); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(151).sizeBytes() == 0); + auto second = mgr.tryReserveWorkingMemoryWithoutEviction(150); + REQUIRE(second.sizeBytes() == 150); + auto moved = std::move(second); + CHECK(second.sizeBytes() == 0); + CHECK(moved.sizeBytes() == 150); + moved.shrinkTo(0); + CHECK(mgr.reservedWorkingMemoryBytes() == 100); + mgr.release(cacheHandle); + mgr.clear(); + mgr.setBudgetBytes(CacheMemoryBudgetManager::defaultBudgetBytes()); +} + +TEST_CASE("CacheMemoryBudgetManager exact working reservations never request eviction", "[CacheMemoryBudgetManager]") +{ + auto& mgr = CacheMemoryBudgetManager::instance(); + mgr.clear(); + mgr.setBudgetBytes(1000); + + int callbackCalls = 0; + int handlerCalls = 0; + auto [directHandle, directEvictions] = mgr.allocate("exact-direct", "entry", 500, [&callbackCalls]() { callbackCalls++; }); + REQUIRE(directEvictions.empty()); + mgr.registerSubsystem("exact-delegated", [&handlerCalls](uint64) { handlerCalls++; }); + auto [delegatedHandle, delegatedEvictions] = mgr.allocate("exact-delegated", "entry", 300, []() {}); + REQUIRE(delegatedEvictions.empty()); + + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(201).sizeBytes() == 0); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(251).sizeBytes() == 0); + CHECK(callbackCalls == 0); + CHECK(handlerCalls == 0); + CHECK(mgr.usedBytes() == 800); + + auto fit = mgr.tryReserveWorkingMemoryWithoutEviction(200); + REQUIRE(fit.sizeBytes() == 200); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(51).sizeBytes() == 0); + CHECK(callbackCalls == 0); + CHECK(handlerCalls == 0); + + mgr.release(directHandle); + mgr.release(delegatedHandle); + mgr.registerSubsystem("exact-delegated", [](uint64) {}); + mgr.clear(); + mgr.setBudgetBytes(CacheMemoryBudgetManager::defaultBudgetBytes()); +} + +TEST_CASE("CacheMemoryBudgetManager exact working reservations count pinned and pending entries", "[CacheMemoryBudgetManager]") +{ + auto& mgr = CacheMemoryBudgetManager::instance(); + mgr.clear(); + mgr.setBudgetBytes(1000); + + CacheMemoryBudgetManager::AllocationOptions options; + options.initiallyPinned = true; + int callbackCalls = 0; + auto [pinnedHandle, pinnedEvictions] = mgr.allocate("exact-pinned", "entry", 900, [&callbackCalls]() { callbackCalls++; }, options); + REQUIRE(pinnedEvictions.empty()); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(101).sizeBytes() == 0); + auto fit = mgr.tryReserveWorkingMemoryWithoutEviction(100); + REQUIRE(fit.sizeBytes() == 100); + mgr.setBudgetBytes(800); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(1).sizeBytes() == 0); + CHECK(callbackCalls == 0); + fit.shrinkTo(0); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(1).sizeBytes() == 0); + mgr.release(pinnedHandle); + + mgr.setBudgetBytes(1000); + int handlerCalls = 0; + mgr.registerSubsystem("exact-pending", [&handlerCalls](uint64) { handlerCalls++; }); + auto [firstHandle, firstEvictions] = mgr.allocate("exact-pending", "first", 600, []() {}); + REQUIRE(firstEvictions.empty()); + auto [secondHandle, secondEvictions] = mgr.allocate("exact-pending", "second", 600, []() {}); + REQUIRE(secondEvictions.empty()); + REQUIRE(handlerCalls == 1); + CHECK(mgr.usedBytes() == 1200); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(1).sizeBytes() == 0); + CHECK(handlerCalls == 1); + mgr.release(firstHandle); + auto afterRelease = mgr.tryReserveWorkingMemoryWithoutEviction(250); + CHECK(afterRelease.sizeBytes() == 250); + mgr.release(secondHandle); + mgr.registerSubsystem("exact-pending", [](uint64) {}); + mgr.clear(); + mgr.setBudgetBytes(CacheMemoryBudgetManager::defaultBudgetBytes()); +} + +TEST_CASE("CacheMemoryBudgetManager exact working reservations recover after budget shrink", "[CacheMemoryBudgetManager]") +{ + auto& mgr = CacheMemoryBudgetManager::instance(); + mgr.clear(); + mgr.setBudgetBytes(1000); + + { + auto existing = mgr.reserveWorkingMemory(250); + REQUIRE(existing.sizeBytes() == 250); + REQUIRE(mgr.usedBytes() == 0); + mgr.setBudgetBytes(800); + REQUIRE(mgr.maximumWorkingMemoryBytes() == 200); + CHECK(mgr.tryReserveWorkingMemoryWithoutEviction(1).sizeBytes() == 0); + CHECK(mgr.reservedWorkingMemoryBytes() == 250); + + existing.shrinkTo(199); + auto remaining = mgr.tryReserveWorkingMemoryWithoutEviction(1); + REQUIRE(remaining.sizeBytes() == 1); + CHECK(mgr.reservedWorkingMemoryBytes() == 200); + } + + CHECK(mgr.reservedWorkingMemoryBytes() == 0); + mgr.setBudgetBytes(CacheMemoryBudgetManager::defaultBudgetBytes()); +} + +TEST_CASE("CacheMemoryBudgetManager concurrent exact working reservations cannot over-admit", "[CacheMemoryBudgetManager]") +{ + auto& mgr = CacheMemoryBudgetManager::instance(); + for(const uint64 cacheBytes : std::array{0, 850}) + { + mgr.clear(); + mgr.setBudgetBytes(1000); + CacheMemoryBudgetManager::AllocationHandle cacheHandle = 0; + if(cacheBytes != 0) + { + auto [handle, evictions] = mgr.allocate("exact-concurrent", "entry", cacheBytes, []() {}); + REQUIRE(evictions.empty()); + cacheHandle = handle; + } + + const uint64 requestBytes = cacheBytes == 0 ? 200 : 100; + std::barrier phaseBarrier(3); + std::array grants = {}; + std::array workers; + for(usize threadIndex = 0; threadIndex < workers.size(); threadIndex++) + { + workers[threadIndex] = std::thread([&, threadIndex]() { + phaseBarrier.arrive_and_wait(); + auto reservation = mgr.tryReserveWorkingMemoryWithoutEviction(requestBytes); + grants[threadIndex] = reservation.sizeBytes(); + phaseBarrier.arrive_and_wait(); + phaseBarrier.arrive_and_wait(); + }); + } + phaseBarrier.arrive_and_wait(); + phaseBarrier.arrive_and_wait(); + CHECK(grants[0] + grants[1] == requestBytes); + CHECK(mgr.reservedWorkingMemoryBytes() == requestBytes); + phaseBarrier.arrive_and_wait(); + for(auto& worker : workers) + { + worker.join(); + } + CHECK(mgr.reservedWorkingMemoryBytes() == 0); + if(cacheBytes != 0) + { + mgr.release(cacheHandle); + } + } + mgr.clear(); + mgr.setBudgetBytes(CacheMemoryBudgetManager::defaultBudgetBytes()); +} diff --git a/test/CheckedFlushTest.cpp b/test/CheckedFlushTest.cpp new file mode 100644 index 0000000000..dce1b8dc70 --- /dev/null +++ b/test/CheckedFlushTest.cpp @@ -0,0 +1,693 @@ +#include "simplnx/Common/Result.hpp" +#include "simplnx/DataStructure/AbstractDataStore.hpp" +#include "simplnx/DataStructure/AbstractListStore.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/DataStructure/DataObject.hpp" +#include "simplnx/DataStructure/DataPath.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/IListStore.hpp" +#include "simplnx/DataStructure/ListStore.hpp" +#include "simplnx/DataStructure/NeighborList.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +/** + * @enum FlushFailure + * @brief Selects an exception from a custom flush operation. + */ +enum class FlushFailure +{ + None, ///< Completes the operation. + Standard, ///< Throws a storage error with backing context. + Unknown, ///< Throws a nonstandard exception. + Allocation ///< Throws a simulated allocation failure. +}; + +/** + * @brief Throws the selected exception while diagnostic memory remains available. + * @param failure Selects the exception category. + */ +void ThrowFlushFailure(FlushFailure failure) +{ + switch(failure) + { + case FlushFailure::None: + return; + case FlushFailure::Standard: + throw std::runtime_error("backing.h5:/values: injected flush failure"); + case FlushFailure::Unknown: + throw 42; + case FlushFailure::Allocation: + throw std::bad_alloc{}; + } +} + +/** + * @class LegacyFlushStore + * @brief Supplies a legacy numeric override for the checked default adapter. + */ +class LegacyFlushStore : public DataStore +{ +public: + /** + * @brief Creates two initialized resident values. + */ + LegacyFlushStore() + : DataStore(ShapeType{2}, ShapeType{1}, int32{0}) + { + } + + ~LegacyFlushStore() override = default; + LegacyFlushStore(const LegacyFlushStore&) = default; + LegacyFlushStore(LegacyFlushStore&&) = default; + LegacyFlushStore& operator=(const LegacyFlushStore&) = delete; + LegacyFlushStore& operator=(LegacyFlushStore&&) = default; + + /** + * @brief Counts the legacy call and throws the selected exception. + */ + void flush() const override + { + ++FlushCalls; + ThrowFlushFailure(Failure); + } + + FlushFailure Failure = FlushFailure::None; + mutable usize FlushCalls = 0; +}; + +/** + * @class ReportingFlushStore + * @brief Supplies numeric diagnostics without calling the legacy operation. + */ +class ReportingFlushStore : public DataStore +{ +public: + /** + * @brief Creates two initialized resident values. + */ + ReportingFlushStore() + : DataStore(ShapeType{2}, ShapeType{1}, int32{0}) + { + } + + ~ReportingFlushStore() override = default; + ReportingFlushStore(const ReportingFlushStore&) = default; + ReportingFlushStore(ReportingFlushStore&&) = default; + ReportingFlushStore& operator=(const ReportingFlushStore&) = delete; + ReportingFlushStore& operator=(ReportingFlushStore&&) = default; + + /** + * @brief Counts calls to the separate legacy operation. + */ + void flush() const override + { + ++LegacyCalls; + } + + /** + * @brief Counts the checked call and returns the selected diagnostics. + * @return The configured result, including warnings. + */ + [[nodiscard]] Result<> flushChecked() const override + { + ++CheckedCalls; + return FlushResult; + } + + Result<> FlushResult; + mutable usize CheckedCalls = 0; + mutable usize LegacyCalls = 0; +}; + +/** + * @class ReportingFlushListStore + * @brief Supplies list diagnostics to a real NeighborList object. + */ +class ReportingFlushListStore : public ListStore +{ +public: + /** + * @brief Creates two empty resident lists. + */ + ReportingFlushListStore() + : ListStore(ShapeType{2}) + { + } + + ~ReportingFlushListStore() override = default; + ReportingFlushListStore(const ReportingFlushListStore&) = default; + ReportingFlushListStore(ReportingFlushListStore&&) = default; + ReportingFlushListStore& operator=(const ReportingFlushListStore&) = delete; + ReportingFlushListStore& operator=(ReportingFlushListStore&&) = delete; + + /** + * @brief Counts the checked call and returns the selected diagnostics. + * @return The configured result, including warnings. + */ + [[nodiscard]] Result<> flushChecked() const override + { + ++CheckedCalls; + return FlushResult; + } + + Result<> FlushResult; + mutable usize CheckedCalls = 0; +}; + +/** + * @class LegacyFlushGroup + * @brief Supplies a legacy object override for the DataObject default adapter. + */ +class LegacyFlushGroup : public DataGroup +{ +public: + /** + * @brief Creates a named group with normal DataObject identity. + * @param dataStructure Owns the group after insertion. + * @param name Names the group. + */ + LegacyFlushGroup(DataStructure& dataStructure, std::string name) + : DataGroup(dataStructure, std::move(name)) + { + } + + ~LegacyFlushGroup() override = default; + LegacyFlushGroup(const LegacyFlushGroup&) = default; + LegacyFlushGroup(LegacyFlushGroup&&) = default; + LegacyFlushGroup& operator=(const LegacyFlushGroup&) = delete; + LegacyFlushGroup& operator=(LegacyFlushGroup&&) = delete; + + /** + * @brief Counts the legacy call and throws the selected exception. + */ + void flush() const override + { + ++FlushCalls; + ThrowFlushFailure(Failure); + } + + FlushFailure Failure = FlushFailure::None; + mutable usize FlushCalls = 0; +}; + +/** + * @class ThrowingCheckedFlushGroup + * @brief Bypasses the default adapter to exercise DataStructure failure handling. + */ +class ThrowingCheckedFlushGroup : public DataGroup +{ +public: + /** + * @brief Creates a named group with normal DataObject identity. + * @param dataStructure Owns the group after insertion. + * @param name Names the group. + */ + ThrowingCheckedFlushGroup(DataStructure& dataStructure, std::string name) + : DataGroup(dataStructure, std::move(name)) + { + } + + ~ThrowingCheckedFlushGroup() override = default; + ThrowingCheckedFlushGroup(const ThrowingCheckedFlushGroup&) = default; + ThrowingCheckedFlushGroup(ThrowingCheckedFlushGroup&&) = default; + ThrowingCheckedFlushGroup& operator=(const ThrowingCheckedFlushGroup&) = delete; + ThrowingCheckedFlushGroup& operator=(ThrowingCheckedFlushGroup&&) = delete; + + /** + * @brief Counts the checked call and throws the selected exception. + * @return The configured result when no exception is selected. + */ + [[nodiscard]] Result<> flushChecked() const override + { + ++CheckedCalls; + ThrowFlushFailure(Failure); + return FlushResult; + } + + FlushFailure Failure = FlushFailure::None; + Result<> FlushResult; + mutable usize CheckedCalls = 0; +}; +} // namespace + +TEST_CASE("CheckedFlush resident defaults succeed", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + const DataStructure& constStructure = dataStructure; + const auto emptyResult = constStructure.flushChecked(); + REQUIRE(emptyResult.valid()); + CHECK(emptyResult.warnings().empty()); + + auto numericStore = std::make_shared>(ShapeType{2}, ShapeType{1}, int32{7}); + auto listStore = std::make_shared>(ShapeType{2}); + listStore->setList(0, std::vector{11, 13}); + const AbstractDataStore& constNumericStore = *numericStore; + const IListStore& constListStore = *listStore; + REQUIRE(constNumericStore.flushChecked().valid()); + REQUIRE(constListStore.flushChecked().valid()); + + const DataStore boolStore(ShapeType{2}, ShapeType{1}, true); + REQUIRE(boolStore.flushChecked().valid()); + const auto* groupPtr = DataGroup::Create(dataStructure, "Group"); + REQUIRE(groupPtr != nullptr); + const auto* arrayPtr = DataArray::Create(dataStructure, "Values", numericStore, groupPtr->getId()); + const auto* listPtr = NeighborList::Create(dataStructure, "Neighbors", listStore, groupPtr->getId()); + REQUIRE(arrayPtr != nullptr); + REQUIRE(listPtr != nullptr); + REQUIRE(groupPtr->flushChecked().valid()); + REQUIRE(arrayPtr->flushChecked().valid()); + REQUIRE(listPtr->flushChecked().valid()); + + const auto result = constStructure.flushChecked(); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + CHECK(numericStore->getValue(0) == 7); + CHECK(listStore->getList(0) == std::vector{11, 13}); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("CheckedFlush adapts legacy numeric exceptions", "[simplnx][CheckedFlush]") +{ + const auto [failure, expectedCode, detail] = GENERATE(Catch::Generators::table( + {{FlushFailure::None, 0, ""}, {FlushFailure::Standard, -6070, "backing.h5:/values"}, {FlushFailure::Unknown, -6070, "unknown"}, {FlushFailure::Allocation, -272, "alloc"}})); + DYNAMIC_SECTION("Exception category " << static_cast(failure)) + { + LegacyFlushStore store; + store.Failure = failure; + const AbstractDataStore& abstractStore = store; + Result<> result; + REQUIRE_NOTHROW(result = abstractStore.flushChecked()); + CHECK(store.FlushCalls == 1); + CHECK(result.warnings().empty()); + if(expectedCode == 0) + { + REQUIRE(result.valid()); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == expectedCode); + CHECK(result.errors()[0].message.find(detail) != std::string::npos); + } + } +} + +TEST_CASE("CheckedFlush adapts legacy object exceptions", "[simplnx][CheckedFlush]") +{ + const auto [failure, expectedCode, detail] = GENERATE(Catch::Generators::table( + {{FlushFailure::None, 0, ""}, {FlushFailure::Standard, -6070, "backing.h5:/values"}, {FlushFailure::Unknown, -6070, "unknown"}, {FlushFailure::Allocation, -272, "alloc"}})); + DYNAMIC_SECTION("Exception category " << static_cast(failure)) + { + DataStructure dataStructure; + auto object = std::make_shared(dataStructure, "LegacyObject"); + REQUIRE(dataStructure.insert(object, DataPath{})); + object->Failure = failure; + const DataObject& abstractObject = *object; + Result<> result; + REQUIRE_NOTHROW(result = abstractObject.flushChecked()); + CHECK(object->FlushCalls == 1); + CHECK(result.warnings().empty()); + if(expectedCode == 0) + { + REQUIRE(result.valid()); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == expectedCode); + CHECK(result.errors()[0].message.find(detail) != std::string::npos); + CHECK(result.errors()[0].message.find("LegacyObject") != std::string::npos); + } + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("CheckedFlush forwards numeric and list diagnostics once", "[simplnx][CheckedFlush]") +{ + const bool useList = GENERATE(false, true); + const bool hasErrors = GENERATE(false, true); + DYNAMIC_SECTION("List " << useList << ", errors " << hasErrors) + { + DataStructure dataStructure; + auto numericStore = std::make_shared(); + auto listStore = std::make_shared(); + const auto* arrayPtr = DataArray::Create(dataStructure, "Values", numericStore); + const auto* listPtr = NeighborList::Create(dataStructure, "Neighbors", listStore); + REQUIRE(arrayPtr != nullptr); + REQUIRE(listPtr != nullptr); + + Result<> reportedResult; + if(hasErrors) + { + reportedResult = MakeErrorResult(-19001, "backing.h5:/values: primary storage error"); + reportedResult.errors().push_back({-272, "backing.h5:/values: memory allocation failed during cleanup"}); + } + reportedResult.warnings().push_back({19001, "backing.h5:/values: first storage warning"}); + reportedResult.warnings().push_back({19002, "backing.h5:/values: second storage warning"}); + numericStore->FlushResult = reportedResult; + listStore->FlushResult = reportedResult; + + const DataObject& object = useList ? static_cast(*listPtr) : static_cast(*arrayPtr); + const auto result = object.flushChecked(); + CHECK(numericStore->CheckedCalls == (useList ? 0 : 1)); + CHECK(listStore->CheckedCalls == (useList ? 1 : 0)); + CHECK(numericStore->LegacyCalls == 0); + CHECK(result.invalid() == hasErrors); + REQUIRE(result.warnings().size() == 2); + CHECK(result.warnings()[0].code == 19001); + CHECK(result.warnings()[0].message.find("first storage warning") != std::string::npos); + CHECK(result.warnings()[1].code == 19002); + CHECK(result.warnings()[1].message.find("second storage warning") != std::string::npos); + if(hasErrors) + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == -19001); + CHECK(result.errors()[0].message.find("backing.h5:/values: primary storage error") != std::string::npos); + CHECK(result.errors()[1].code == -272); + CHECK(result.errors()[1].message.find("memory allocation failed during cleanup") != std::string::npos); + } + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("CheckedFlush stops at the first failed object and keeps all of its paths", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + const auto* firstParentPtr = DataGroup::Create(dataStructure, "First"); + const auto* secondParentPtr = DataGroup::Create(dataStructure, "Second"); + REQUIRE(firstParentPtr != nullptr); + REQUIRE(secondParentPtr != nullptr); + auto warningStore = std::make_shared(); + warningStore->FlushResult = MakeWarningVoidResult(19100, "earlier warning"); + const auto* warningArrayPtr = DataArray::Create(dataStructure, "Earlier", warningStore); + REQUIRE(warningArrayPtr != nullptr); + auto firstStore = std::make_shared(); + firstStore->FlushResult = MakeErrorResult(-19101, "backing.h5:/first: primary failure"); + firstStore->FlushResult.errors().push_back({-272, "backing.h5:/first: memory allocation failed during cleanup"}); + firstStore->FlushResult.warnings().push_back({19101, "first warning"}); + firstStore->FlushResult.warnings().push_back({19102, "second warning"}); + const auto* firstArrayPtr = DataArray::Create(dataStructure, "Values", firstStore, firstParentPtr->getId()); + REQUIRE(firstArrayPtr != nullptr); + REQUIRE(dataStructure.setAdditionalParent(firstArrayPtr->getId(), secondParentPtr->getId())); + REQUIRE(firstArrayPtr->getDataPaths().size() == 2); + REQUIRE(warningArrayPtr->getId() < firstArrayPtr->getId()); + + auto laterStore = std::make_shared(); + laterStore->FlushResult = MakeErrorResult(-19103, "backing.h5:/later: later failure"); + laterStore->FlushResult.warnings().push_back({19103, "later warning"}); + const auto* laterArrayPtr = DataArray::Create(dataStructure, "ALater", laterStore); + REQUIRE(laterArrayPtr != nullptr); + REQUIRE(firstArrayPtr->getId() < laterArrayPtr->getId()); + + const DataStructure& constStructure = dataStructure; + const auto result = constStructure.flushChecked(); + REQUIRE(result.invalid()); + CHECK(warningStore->CheckedCalls == 1); + CHECK(firstStore->CheckedCalls == 1); + // The first failure ends the traversal; later objects keep their unflushed state. + CHECK(laterStore->CheckedCalls == 0); + CHECK(warningStore->LegacyCalls == 0); + CHECK(firstStore->LegacyCalls == 0); + CHECK(laterStore->LegacyCalls == 0); + REQUIRE(result.errors().size() == 2); + CHECK(result.errors()[0].code == -19101); + CHECK(result.errors()[1].code == -272); + for(usize errorIdx = 0; errorIdx < 2; ++errorIdx) + { + CHECK(result.errors()[errorIdx].message.find("backing.h5:/first") != std::string::npos); + CHECK(result.errors()[errorIdx].message.find("First/Values") != std::string::npos); + CHECK(result.errors()[errorIdx].message.find("Second/Values") != std::string::npos); + CHECK(result.errors()[errorIdx].message.find(std::to_string(firstArrayPtr->getId())) != std::string::npos); + } + CHECK(result.errors()[0].message.find("primary failure") != std::string::npos); + CHECK(result.errors()[1].message.find("memory allocation failed during cleanup") != std::string::npos); + REQUIRE(result.warnings().size() == 3); + CHECK(result.warnings()[0].code == 19100); + CHECK(result.warnings()[0].message.find("earlier warning") != std::string::npos); + CHECK(result.warnings()[0].message.find("Earlier") != std::string::npos); + CHECK(result.warnings()[1].code == 19101); + CHECK(result.warnings()[1].message.find("first warning") != std::string::npos); + CHECK(result.warnings()[2].code == 19102); + CHECK(result.warnings()[2].message.find("second warning") != std::string::npos); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("CheckedFlush forwarders preserve invalid results with no errors", "[simplnx][CheckedFlush]") +{ + const bool useList = GENERATE(false, true); + DYNAMIC_SECTION("List " << useList) + { + DataStructure dataStructure; + auto numericStore = std::make_shared(); + auto listStore = std::make_shared(); + const auto* arrayPtr = DataArray::Create(dataStructure, "Values", numericStore); + const auto* listPtr = NeighborList::Create(dataStructure, "Neighbors", listStore); + REQUIRE(arrayPtr != nullptr); + REQUIRE(listPtr != nullptr); + Result<> emptyFailure; + emptyFailure.m_Expected = nonstd::make_unexpected(ErrorCollection{}); + emptyFailure.warnings().push_back({19501, "backing.h5:/values: retained warning"}); + REQUIRE(emptyFailure.invalid()); + REQUIRE(emptyFailure.errors().empty()); + numericStore->FlushResult = emptyFailure; + listStore->FlushResult = emptyFailure; + + const DataObject& object = useList ? static_cast(*listPtr) : static_cast(*arrayPtr); + const auto result = object.flushChecked(); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -6070); + CHECK(result.errors()[0].message.find(useList ? "Neighbors" : "Values") != std::string::npos); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings()[0].code == 19501); + CHECK(result.warnings()[0].message.find("backing.h5:/values: retained warning") != std::string::npos); + CHECK(numericStore->CheckedCalls == (useList ? 0 : 1)); + CHECK(listStore->CheckedCalls == (useList ? 1 : 0)); + CHECK(numericStore->LegacyCalls == 0); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("CheckedFlush traversal preserves invalid custom objects with no errors", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + auto object = std::make_shared(dataStructure, "EmptyFailure"); + object->FlushResult.m_Expected = nonstd::make_unexpected(ErrorCollection{}); + object->FlushResult.warnings().push_back({19601, "empty failure warning"}); + REQUIRE(object->FlushResult.invalid()); + REQUIRE(object->FlushResult.errors().empty()); + REQUIRE(dataStructure.insert(object, DataPath{})); + auto laterStore = std::make_shared(); + laterStore->FlushResult = MakeWarningVoidResult(19602, "later warning"); + const auto* laterArrayPtr = DataArray::Create(dataStructure, "Later", laterStore); + REQUIRE(laterArrayPtr != nullptr); + REQUIRE(object->getId() < laterArrayPtr->getId()); + + const auto result = std::as_const(dataStructure).flushChecked(); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -6070); + CHECK(result.errors()[0].message.find("EmptyFailure") != std::string::npos); + CHECK(result.errors()[0].message.find(std::to_string(object->getId())) != std::string::npos); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings()[0].code == 19601); + CHECK(result.warnings()[0].message.find("empty failure warning") != std::string::npos); + CHECK(object->CheckedCalls == 1); + CHECK(laterStore->CheckedCalls == 0); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("CheckedFlush calls a shared store once for each distinct object", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + auto store = std::make_shared(); + store->FlushResult = MakeWarningVoidResult(19201, "shared backing warning"); + REQUIRE(DataArray::Create(dataStructure, "First", store) != nullptr); + REQUIRE(DataArray::Create(dataStructure, "Second", store) != nullptr); + const auto result = std::as_const(dataStructure).flushChecked(); + REQUIRE(result.valid()); + CHECK(store->CheckedCalls == 2); + CHECK(store->LegacyCalls == 0); + REQUIRE(result.warnings().size() == 2); + CHECK(result.warnings()[0].code == 19201); + CHECK(result.warnings()[1].code == 19201); + CHECK(result.warnings()[0].message.find("shared backing warning") != std::string::npos); + CHECK(result.warnings()[1].message.find("shared backing warning") != std::string::npos); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("CheckedFlush stops at the first failed object before custom checked exceptions", "[simplnx][CheckedFlush]") +{ + const auto [failure, expectedCode, detail] = GENERATE(Catch::Generators::table( + {{FlushFailure::Standard, -6070, "backing.h5:/values"}, {FlushFailure::Unknown, -6070, "unknown"}, {FlushFailure::Allocation, -272, "alloc"}})); + DYNAMIC_SECTION("Exception category " << static_cast(failure)) + { + DataStructure dataStructure; + auto firstStore = std::make_shared(); + firstStore->FlushResult = MakeErrorResult(-19301, "first ordinary failure"); + firstStore->FlushResult.warnings().push_back({19301, "first warning"}); + const auto* firstArrayPtr = DataArray::Create(dataStructure, "First", firstStore); + REQUIRE(firstArrayPtr != nullptr); + auto throwingObject = std::make_shared(dataStructure, "ThrowingObject"); + REQUIRE(dataStructure.insert(throwingObject, DataPath{})); + throwingObject->Failure = failure; + auto laterStore = std::make_shared(); + laterStore->FlushResult = MakeErrorResult(-19302, "later ordinary failure"); + laterStore->FlushResult.warnings().push_back({19302, "later warning"}); + const auto* laterArrayPtr = DataArray::Create(dataStructure, "Later", laterStore); + REQUIRE(laterArrayPtr != nullptr); + REQUIRE(firstArrayPtr->getId() < throwingObject->getId()); + REQUIRE(throwingObject->getId() < laterArrayPtr->getId()); + + Result<> result; + REQUIRE_NOTHROW(result = std::as_const(dataStructure).flushChecked()); + REQUIRE(result.invalid()); + CHECK(firstStore->CheckedCalls == 1); + CHECK(throwingObject->CheckedCalls == 0); + CHECK(laterStore->CheckedCalls == 0); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -19301); + CHECK(result.errors()[0].message.find("first ordinary failure") != std::string::npos); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings()[0].code == 19301); + CHECK(result.warnings()[0].message.find("first warning") != std::string::npos); + + // With the ordinary failure repaired, the next flush stops at the throwing object instead. + firstStore->FlushResult = MakeWarningVoidResult(19303, "repaired warning"); + REQUIRE_NOTHROW(result = std::as_const(dataStructure).flushChecked()); + REQUIRE(result.invalid()); + CHECK(firstStore->CheckedCalls == 2); + CHECK(throwingObject->CheckedCalls == 1); + CHECK(laterStore->CheckedCalls == 0); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == expectedCode); + CHECK(result.errors()[0].message.find(detail) != std::string::npos); + CHECK(result.errors()[0].message.find("ThrowingObject") != std::string::npos); + CHECK(result.errors()[0].message.find(std::to_string(throwingObject->getId())) != std::string::npos); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings()[0].code == 19303); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("CheckedFlush reports a moved-from numeric store without changing object identity", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + auto sourceStore = std::make_shared>(ShapeType{2}, ShapeType{1}, int32{7}); + auto destinationStore = std::make_shared>(ShapeType{2}, ShapeType{1}, int32{11}); + auto* sourcePtr = DataArray::Create(dataStructure, "Source", sourceStore); + auto* destinationPtr = DataArray::Create(dataStructure, "Destination", destinationStore); + REQUIRE(sourcePtr != nullptr); + REQUIRE(destinationPtr != nullptr); + const auto sourceId = sourcePtr->getId(); + const auto destinationId = destinationPtr->getId(); + + *destinationPtr = std::move(*sourcePtr); + REQUIRE(sourcePtr->getDataStore() == nullptr); + REQUIRE(destinationPtr->getDataStore() == sourceStore.get()); + CHECK(sourcePtr->getId() == sourceId); + CHECK(destinationPtr->getId() == destinationId); + CHECK(dataStructure.getData(DataPath({"Source"})) == sourcePtr); + CHECK(dataStructure.getData(DataPath({"Destination"})) == destinationPtr); + const auto directResult = std::as_const(*sourcePtr).flushChecked(); + REQUIRE(directResult.invalid()); + REQUIRE_FALSE(directResult.errors().empty()); + CHECK(directResult.errors()[0].message.find("Source") != std::string::npos); + const auto result = std::as_const(dataStructure).flushChecked(); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].message.find("Source") != std::string::npos); + CHECK(sourceStore->getValue(0) == 7); + + REQUIRE(sourcePtr->setDataStore(sourceStore).valid()); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("CheckedFlush reports a missing list store from the public setter", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + auto store = std::make_shared>(ShapeType{2}); + auto* listPtr = NeighborList::Create(dataStructure, "Neighbors", store); + REQUIRE(listPtr != nullptr); + listPtr->setStore(std::shared_ptr>{}); + REQUIRE(listPtr->getIListStore() == nullptr); + const auto directResult = std::as_const(*listPtr).flushChecked(); + REQUIRE(directResult.invalid()); + REQUIRE_FALSE(directResult.errors().empty()); + CHECK(directResult.errors()[0].message.find("Neighbors") != std::string::npos); + const auto result = std::as_const(dataStructure).flushChecked(); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].message.find("Neighbors") != std::string::npos); + + listPtr->setStore(store); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("CheckedFlush preserves legacy void failure behavior", "[simplnx][CheckedFlush]") +{ + const auto failure = GENERATE(FlushFailure::Standard, FlushFailure::Unknown, FlushFailure::Allocation); + DYNAMIC_SECTION("Exception category " << static_cast(failure)) + { + DataStructure dataStructure; + auto firstStore = std::make_shared(); + firstStore->Failure = failure; + auto laterStore = std::make_shared(); + REQUIRE(DataArray::Create(dataStructure, "First", firstStore) != nullptr); + REQUIRE(DataArray::Create(dataStructure, "Later", laterStore) != nullptr); + const DataStructure& constStructure = dataStructure; + switch(failure) + { + case FlushFailure::Standard: + REQUIRE_THROWS_AS(constStructure.flush(), std::runtime_error); + break; + case FlushFailure::Unknown: + REQUIRE_THROWS_AS(constStructure.flush(), int); + break; + case FlushFailure::Allocation: + REQUIRE_THROWS_AS(constStructure.flush(), std::bad_alloc); + break; + case FlushFailure::None: + FAIL("The legacy failure test requires an exception."); + } + CHECK(firstStore->FlushCalls == 1); + CHECK(laterStore->LegacyCalls == 0); + CHECK(laterStore->CheckedCalls == 0); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +} + +TEST_CASE("CheckedFlush leaves legacy success separate from checked overrides", "[simplnx][CheckedFlush]") +{ + DataStructure dataStructure; + auto numericStore = std::make_shared(); + numericStore->FlushResult = MakeErrorResult(-19401, "checked numeric failure"); + auto listStore = std::make_shared(); + listStore->FlushResult = MakeErrorResult(-19402, "checked list failure"); + REQUIRE(DataArray::Create(dataStructure, "Values", numericStore) != nullptr); + REQUIRE(NeighborList::Create(dataStructure, "Neighbors", listStore) != nullptr); + REQUIRE_NOTHROW(std::as_const(dataStructure).flush()); + CHECK(numericStore->LegacyCalls == 1); + CHECK(numericStore->CheckedCalls == 0); + CHECK(listStore->CheckedCalls == 0); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} diff --git a/test/ChunkShapePolicyTest.cpp b/test/ChunkShapePolicyTest.cpp index 7e0305b091..fce0faf6d0 100644 --- a/test/ChunkShapePolicyTest.cpp +++ b/test/ChunkShapePolicyTest.cpp @@ -2,9 +2,61 @@ #include +#include + using namespace nx::core; using namespace nx::core::HDF5; +TEST_CASE("ChunkShapePolicy: caller storage matches independent shapes", "[ChunkShapePolicy]") +{ + const auto regime = GENERATE(ChunkShapeRegime::BundleOuterSlabs, ChunkShapeRegime::PinSlowestDim); + const ShapeType dims{64, 2048, 512}; + const ShapeType originalDims = dims; + std::array output{73, 73, 73, 73}; + std::array scratch{91, 91, 91, 91}; + const ChunkShapeOptions options{k_TargetChunkBytes, regime}; + REQUIRE(ComputeChunkShapeInto(dims, 1, 4, options, output, scratch)); + CHECK(output == std::array{1, 512, 512, 73}); + CHECK(scratch.back() == 91); + CHECK(computeChunkShape(dims, 1, 4, options) == ShapeType{1, 512, 512}); + output.fill(73); + scratch.fill(91); + REQUIRE_FALSE(ComputeChunkShapeInto(dims, 1, 4, options, {output.data(), 2}, scratch)); + CHECK(output == std::array{73, 73, 73, 73}); + CHECK(scratch == std::array{91, 91, 91, 91}); + CHECK(dims == originalDims); + const bool requiresSuffix = regime == ChunkShapeRegime::BundleOuterSlabs; + CHECK(ComputeChunkShapeInto(dims, 1, 4, options, output, {}) == !requiresSuffix); + if(requiresSuffix) + { + CHECK(output == std::array{73, 73, 73, 73}); + CHECK(scratch == std::array{91, 91, 91, 91}); + } + CHECK(dims == originalDims); + REQUIRE(ComputeChunkShapeInto({}, 1, 4, options, {}, {})); + REQUIRE(ComputeChunkShapeInto(ShapeType{137, 1, 1}, 1, 4, options, output, scratch)); + CHECK(output[0] == (requiresSuffix ? 137 : 1)); + CHECK(output[1] == 1); + CHECK(output[2] == 1); + const ShapeType oneElementDims{7}; + REQUIRE(ComputeChunkShapeInto(oneElementDims, 3, 4, {12, regime}, output, scratch)); + CHECK(output[0] == 1); + CHECK(oneElementDims == ShapeType{7}); +} + +namespace +{ +ShapeType ComputeAndCheckScratch(const ShapeType& dimensions, usize components, usize bytes, const ChunkShapeOptions& options) +{ + const auto expected = computeChunkShape(dimensions, components, bytes, options); + ShapeType output(dimensions.size(), 73); + ShapeType scratch(dimensions.size(), 91); + REQUIRE(ComputeChunkShapeInto(dimensions, components, bytes, options, output, scratch)); + CHECK(output == expected); + return output; +} +} // namespace + // All expected shapes below are derived by hand from k_TargetChunkBytes = 1 MiB (1048576 bytes). // suffixBytes[i] = (elementByteSize * numComponents) * product(dims[i+1..]); rowBytes for the // PinSlowestDim regime is (elementByteSize * numComponents) * product(dims[firstRowDim..]). @@ -17,7 +69,7 @@ TEST_CASE("ChunkShapePolicy: BundleOuterSlabs regime", "[ChunkShapePolicy]") { // dims={64,2048,512}, elem=4, nc=1. suffix[0]=4*2048*512=4194304 >= 1MiB -> 1; // suffix[1]=4*512=2048 < 1MiB -> rows=1048576/2048=512, clamp 2048 -> 512. - const ShapeType chunk = computeChunkShape({64, 2048, 512}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({64, 2048, 512}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{1, 512, 512}); } @@ -25,7 +77,7 @@ TEST_CASE("ChunkShapePolicy: BundleOuterSlabs regime", "[ChunkShapePolicy]") { // dims={100,16,16}, elem=4, nc=1. suffix[0]=4*16*16=1024 < 1MiB -> // rows=1048576/1024=1024, clamp 100 -> 100. All 100 outer slabs bundled. - const ShapeType chunk = computeChunkShape({100, 16, 16}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({100, 16, 16}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{100, 16, 16}); } @@ -33,7 +85,7 @@ TEST_CASE("ChunkShapePolicy: BundleOuterSlabs regime", "[ChunkShapePolicy]") { // dims={1000,524288}, elem=4, nc=1. suffix[0]=4*524288=2097152 >= 1MiB -> 1; // suffix[1]=4 < 1MiB -> rows=1048576/4=262144, clamp 524288 -> 262144. - const ShapeType chunk = computeChunkShape({1000, 524288}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({1000, 524288}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{1, 262144}); } @@ -41,7 +93,7 @@ TEST_CASE("ChunkShapePolicy: BundleOuterSlabs regime", "[ChunkShapePolicy]") { // dims={1000000}, elem=4, nc=1. suffix[0]=4 < 1MiB -> // rows=1048576/4=262144, clamp 1000000 -> 262144. - const ShapeType chunk = computeChunkShape({1000000}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({1000000}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{262144}); } @@ -50,13 +102,13 @@ TEST_CASE("ChunkShapePolicy: BundleOuterSlabs regime", "[ChunkShapePolicy]") // dims={64,512,512,3} (component dim folded into dims), elem=4, nc=1. // suffix[3]=4, suffix[2]=4*3=12, suffix[1]=12*512=6144, suffix[0]=6144*512=3145728 >= 1MiB -> 1; // suffix[1]=6144 < 1MiB -> rows=1048576/6144=170, clamp 512 -> 170. - const ShapeType chunk = computeChunkShape({64, 512, 512, 3}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({64, 512, 512, 3}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{1, 170, 512, 3}); } SECTION("empty dims -> empty shape") { - REQUIRE(computeChunkShape({}, /*numComponents=*/1, /*elementByteSize=*/4, opts).empty()); + REQUIRE(ComputeAndCheckScratch({}, /*numComponents=*/1, /*elementByteSize=*/4, opts).empty()); } } @@ -68,7 +120,7 @@ TEST_CASE("ChunkShapePolicy: PinSlowestDim regime", "[ChunkShapePolicy]") { // dims={64,2048,512}, elem=4, nc=1. rowBytes(firstRowDim=2)=4*512=2048 -> // rows=1048576/2048=512, clamp dims[1]=2048 -> 512. - const ShapeType chunk = computeChunkShape({64, 2048, 512}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({64, 2048, 512}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{1, 512, 512}); } @@ -76,7 +128,7 @@ TEST_CASE("ChunkShapePolicy: PinSlowestDim regime", "[ChunkShapePolicy]") { // dims={1000000}, elem=4, nc=1. rowBytes(firstRowDim=1)=4 -> // rows=1048576/4=262144, clamp dims[0]=1000000 -> 262144. - const ShapeType chunk = computeChunkShape({1000000}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({1000000}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{262144}); } @@ -84,7 +136,7 @@ TEST_CASE("ChunkShapePolicy: PinSlowestDim regime", "[ChunkShapePolicy]") { // dims={1000,512}, elem=4, nc=1. rowBytes(firstRowDim=1)=4*512=2048 -> // rows=1048576/2048=512, clamp dims[0]=1000 -> 512. - const ShapeType chunk = computeChunkShape({1000, 512}, /*numComponents=*/1, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({1000, 512}, /*numComponents=*/1, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{512, 512}); } @@ -92,7 +144,7 @@ TEST_CASE("ChunkShapePolicy: PinSlowestDim regime", "[ChunkShapePolicy]") { // tupleShape={64,512,512}, componentShape product=3, elem=4 -> unitBytes=12. // rowBytes(firstRowDim=2)=12*512=6144 -> rows=1048576/6144=170, clamp 512 -> 170. - const ShapeType chunk = computeChunkShape({64, 512, 512}, /*numComponents=*/3, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({64, 512, 512}, /*numComponents=*/3, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{1, 170, 512}); } @@ -100,12 +152,12 @@ TEST_CASE("ChunkShapePolicy: PinSlowestDim regime", "[ChunkShapePolicy]") { // tupleShape={1000,256}, componentShape product=4, elem=4 -> unitBytes=16. // rowBytes(firstRowDim=1)=16*256=4096 -> rows=1048576/4096=256, clamp 1000 -> 256. - const ShapeType chunk = computeChunkShape({1000, 256}, /*numComponents=*/4, /*elementByteSize=*/4, opts); + const ShapeType chunk = ComputeAndCheckScratch({1000, 256}, /*numComponents=*/4, /*elementByteSize=*/4, opts); REQUIRE(chunk == ShapeType{256, 256}); } SECTION("empty dims -> empty shape") { - REQUIRE(computeChunkShape({}, /*numComponents=*/1, /*elementByteSize=*/4, opts).empty()); + REQUIRE(ComputeAndCheckScratch({}, /*numComponents=*/1, /*elementByteSize=*/4, opts).empty()); } } diff --git a/test/ConversionTest/CMakeLists.txt b/test/ConversionTest/CMakeLists.txt index 780cf941cf..f3ef1045e9 100644 --- a/test/ConversionTest/CMakeLists.txt +++ b/test/ConversionTest/CMakeLists.txt @@ -19,6 +19,7 @@ include(Catch) add_executable(conversion_test ${SIMPLNX_TEST_DIRS_HEADER} conversion_test_main.cpp + ../ProfileIsolationContract.cpp SIMPLMapInventoryTest.cpp ) @@ -58,3 +59,11 @@ target_include_directories(conversion_test PRIVATE ${SIMPLNX_GENERATED_DIR}) catch_discover_tests(conversion_test) + +add_test(NAME "conversion_test_profile_isolation" + COMMAND ${CMAKE_COMMAND} + "-DPROFILE_TEST_EXECUTABLE=$" + "-DPROFILE_TEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/ProfileIsolation" + -DPROFILE_RUNNER_ONLY=TRUE + -P "${simplnx_SOURCE_DIR}/test/VerifyTestProfileLifecycle.cmake") +set_tests_properties("conversion_test_profile_isolation" PROPERTIES RUN_SERIAL TRUE TIMEOUT 60) diff --git a/test/ConversionTest/conversion_test_main.cpp b/test/ConversionTest/conversion_test_main.cpp index 48425c5326..6b322c3f17 100644 --- a/test/ConversionTest/conversion_test_main.cpp +++ b/test/ConversionTest/conversion_test_main.cpp @@ -1,4 +1,11 @@ // Catch2 recommends placing these lines by themselves in a translation unit // which will help reduce unnecessary recompilations of the expensive Catch main -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_RUNNER #include + +#include "simplnx/UnitTest/OwnedCatchRunner.hpp" + +int main(int argc, char* argv[]) +{ + return nx::core::UnitTest::RunOwnedCatchSession(argc, argv); +} diff --git a/test/DataArrayTest.cpp b/test/DataArrayTest.cpp index d7faf7021d..df1e728808 100644 --- a/test/DataArrayTest.cpp +++ b/test/DataArrayTest.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -2302,8 +2303,18 @@ TEST_CASE("StorageFormatPlan: ArrayCreation preflight records plans without valu TEST_CASE("StorageFormatPlan: value helper preserves nullable factory failures", "[simplnx][DataArray][StorageFormatPlan]") { const std::string format = "value-helper-invalid-factory"; - const auto failure = GENERATE(0, 1); + const auto failure = GENERATE(0, 1, 2); + CAPTURE(failure); auto selectedFactoryCalls = std::make_shared(0); + auto fallbackMemoryFactoryCalls = std::make_shared(0); + + auto& collection = Application::GetOrCreateInstance()->getIOCollection(); + auto memoryEntry = std::find_if(collection.begin(), collection.end(), [](const auto& entry) { return entry.first == Preferences::k_InMemoryFormat.str(); }); + REQUIRE(memoryEntry != collection.end()); + const auto savedMemoryManager = memoryEntry->second; + auto restoreMemoryManager = MakeScopeGuard([memoryEntry, savedMemoryManager]() noexcept { memoryEntry->second = savedMemoryManager; }); + memoryEntry->second = std::make_shared(savedMemoryManager, fallbackMemoryFactoryCalls); + auto manager = std::make_shared( format, [failure, selectedFactoryCalls](DataType, const ShapeType& tuples, const ShapeType& components, const std::optional&, DataStoreInitializationMode) -> std::unique_ptr { @@ -2312,9 +2323,13 @@ TEST_CASE("StorageFormatPlan: value helper preserves nullable factory failures", { return nullptr; } - return std::make_unique>(tuples, components, 0.0F); + if(failure == 1) + { + return std::make_unique>(tuples, components, 0.0F); + } + return std::make_unique>(tuples, components, 0); }); - auto managerRegistrationResult = Application::GetOrCreateInstance()->getIOCollection().addIOManager(manager); + auto managerRegistrationResult = collection.addIOManager(manager); SIMPLNX_RESULT_REQUIRE_VALID(managerRegistrationResult); DataStructure ds; @@ -2327,6 +2342,7 @@ TEST_CASE("StorageFormatPlan: value helper preserves nullable factory failures", REQUIRE(store == nullptr); REQUIRE(*selectedFactoryCalls == 1); + REQUIRE(*fallbackMemoryFactoryCalls == 0); } TEST_CASE("StorageFormatPlan: canonical memory factory identity is accepted", "[simplnx][DataArray][StorageFormatPlan]") @@ -2564,3 +2580,164 @@ TEST_CASE("Large metadata array copies resolve exact logical bytes without alloc SUCCEED("A 100 GB logical array exceeds this host's supported usize range"); } } + +TEMPLATE_TEST_CASE("DataStore common initialization API changes future values only", "[simplnx][DataStore][GrowthInitialization]", int32, bool) +{ + using T = TestType; + static_assert(GetMudflap() == true); + DataStore concrete({2}, {2}, static_cast(7)); + AbstractDataStore& store = concrete; + CHECK(store.getInitValue() == std::optional{static_cast(7)}); + const std::array prefix{T{0}, T{1}, T{1}, T{0}}; + REQUIRE(store.copyFromBuffer(0, {prefix.data(), prefix.size()}).valid()); + store.setInitValue(T{0}); + CHECK(store.getInitValue() == std::optional{T{0}}); + store.setInitValue(std::optional{T{1}}); + CHECK(store.getInitValue() == std::optional{T{1}}); + std::array unchanged{}; + REQUIRE(store.copyIntoBuffer(0, {unchanged.data(), unchanged.size()}).valid()); + CHECK(unchanged == prefix); + REQUIRE(store.resizeTuples({3}).valid()); + std::array actual{}; + REQUIRE(store.copyIntoBuffer(0, {actual.data(), actual.size()}).valid()); + CHECK(actual == std::array{T{0}, T{1}, T{1}, T{0}, T{1}, T{1}}); + store.setInitValue(std::nullopt); + CHECK_FALSE(store.getInitValue().has_value()); + REQUIRE(store.resizeTuples({4}).valid()); + const T diagnostic = [] { + if constexpr(std::is_same_v) + { + return true; + } + else + { + return T{-1414812757}; + } + }(); + std::array grown{}; + REQUIRE(store.copyIntoBuffer(0, {grown.data(), grown.size()}).valid()); + CHECK(grown == std::array{T{0}, T{1}, T{1}, T{0}, T{1}, T{1}, diagnostic, diagnostic}); +} + +TEST_CASE("DataStore copy and move retain optional initialization", "[simplnx][DataStore][GrowthInitialization]") +{ + const bool missingInitializer = GENERATE(false, true); + const int operation = GENERATE(0, 1, 2); + DYNAMIC_SECTION("missing=" << missingInitializer << " operation=" << operation) + { + const std::optional initial = missingInitializer ? std::nullopt : std::optional{7}; + DataStore source({2}, {2}, initial); + const std::array prefix{10, 11, 20, 21}; + REQUIRE(source.copyFromBuffer(0, {prefix.data(), prefix.size()}).valid()); + std::unique_ptr> target; + if(operation == 0) + { + target = std::make_unique>(source); + target->setValue(0, 99); + CHECK(source.getValue(0) == 10); + target->setValue(0, 10); + } + else if(operation == 1) + { + target = std::make_unique>(std::move(source)); + } + else + { + target = std::make_unique>(ShapeType{1}, ShapeType{1}, int32{-9}); + *target = std::move(source); + } + CHECK(target->getInitValue() == initial); + REQUIRE(target->resizeTuples({3}).valid()); + const int32 tail = missingInitializer ? -1414812757 : 7; + std::array actual{}; + REQUIRE(target->copyIntoBuffer(0, {actual.data(), actual.size()}).valid()); + CHECK(actual == std::array{10, 11, 20, 21, tail, tail}); + } +} + +TEMPLATE_TEST_CASE("DataStore owned buffers retain an engaged diagnostic initializer", "[simplnx][DataStore][GrowthInitialization]", int32, bool) +{ + using T = TestType; + auto buffer = std::make_unique(2); + buffer[0] = T{0}; + buffer[1] = T{1}; + DataStore store(std::move(buffer), {2}, {1}); + const T diagnostic = [] { + if constexpr(std::is_same_v) + { + return true; + } + else + { + return T{-1414812757}; + } + }(); + CHECK(store.getInitValue() == std::optional{diagnostic}); + REQUIRE(store.resizeTuples({3}).valid()); + std::array actual{}; + REQUIRE(store.copyIntoBuffer(0, {actual.data(), actual.size()}).valid()); + CHECK(actual == std::array{T{0}, T{1}, diagnostic}); +} + +TEST_CASE("Numeric placeholders retain initialization through copy and move", "[simplnx][DataStore][GrowthInitialization]") +{ + const bool missingInitializer = GENERATE(false, true); + const int operation = GENERATE(0, 1, 2); + DYNAMIC_SECTION("missing=" << missingInitializer << " operation=" << operation) + { + auto sourceResult = EmptyDataStore::Create({8}, {2}, ""); + REQUIRE(sourceResult.valid()); + auto source = std::move(sourceResult.value()); + CHECK(source->getInitValue() == std::optional{0}); + const std::optional initial = missingInitializer ? std::nullopt : std::optional{7}; + source->setInitValue(initial); + std::unique_ptr copy; + if(operation == 0) + { + copy = std::make_unique>(*source); + } + else if(operation == 1) + { + copy = std::make_unique>(std::move(*source)); + } + else + { + copy = source->deepCopy(""); + } + const auto* typedCopy = dynamic_cast*>(copy.get()); + REQUIRE(typedCopy != nullptr); + CHECK(typedCopy->getStoreType() == IDataStore::StoreType::Empty); + CHECK(typedCopy->getPlannedStoreType() == IDataStore::StoreType::InMemory); + CHECK(typedCopy->getTupleShape() == ShapeType{8}); + CHECK(typedCopy->getComponentShape() == ShapeType{2}); + CHECK(typedCopy->getInitValue() == initial); + CHECK(typedCopy->getDataFormat().empty()); + } +} + +TEST_CASE("DataStore resident value copies retain destination initialization", "[simplnx][DataStore][GrowthInitialization]") +{ + const int operation = GENERATE(0, 1, 2); + DataStore source({2}, {2}, int32{-1}); + DataStore destination({2}, {2}, int32{7}); + const std::array prefix{10, 11, 20, 21}; + REQUIRE(source.copyFromBuffer(0, {prefix.data(), prefix.size()}).valid()); + if(operation == 0) + { + REQUIRE(destination.copyFromBuffer(0, {prefix.data(), prefix.size()}).valid()); + } + else if(operation == 1) + { + REQUIRE(destination.copyFrom(0, source, 0, 2).valid()); + } + else + { + REQUIRE(destination.copy(source)); + } + CHECK(destination.getInitValue() == std::optional{7}); + CHECK(source.getInitValue() == std::optional{-1}); + REQUIRE(destination.resizeTuples({3}).valid()); + std::array actual{}; + REQUIRE(destination.copyIntoBuffer(0, {actual.data(), actual.size()}).valid()); + CHECK(actual == std::array{10, 11, 20, 21, 7, 7}); +} diff --git a/test/DataStorageModeMigrationTest.cpp b/test/DataStorageModeMigrationTest.cpp index 728bb91079..030e229f81 100644 --- a/test/DataStorageModeMigrationTest.cpp +++ b/test/DataStorageModeMigrationTest.cpp @@ -3,8 +3,14 @@ #include #include +#include +#include #include #include +#include +#include +#include +#include using namespace nx::core; namespace fs = std::filesystem; @@ -163,3 +169,409 @@ TEST_CASE("Cache memory budget preference migrates to the unambiguous key", "[Co verifyMigration({{"memory_budget_bytes", k_LegacyBudget}, {"cache_memory_budget_bytes", k_CanonicalBudget}}, k_CanonicalBudget); } } + +namespace +{ +/** + * @struct ThresholdPreferenceFiles + * @brief Owns unique files without reading or changing the process preferences. + */ +struct ThresholdPreferenceFiles +{ + std::filesystem::path root; + std::filesystem::path input; + std::filesystem::path output; + + ThresholdPreferenceFiles() + { + const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); + for(usize attempt = 0; attempt < 1000; ++attempt) + { + const auto candidate = std::filesystem::temp_directory_path() / ("simplnx_threshold_preferences_" + std::to_string(stamp) + "_" + std::to_string(attempt)); + if(std::filesystem::create_directory(candidate)) + { + root = candidate; + input = root / "input.json"; + output = root / "output.json"; + return; + } + } + throw std::runtime_error("Could not create a unique temporary directory for threshold preference tests."); + } + + ~ThresholdPreferenceFiles() + { + std::error_code errorCode; + std::filesystem::remove(input, errorCode); + std::filesystem::remove(output, errorCode); + std::filesystem::remove(root, errorCode); + } + + ThresholdPreferenceFiles(const ThresholdPreferenceFiles&) = delete; + ThresholdPreferenceFiles(ThresholdPreferenceFiles&&) = delete; + ThresholdPreferenceFiles& operator=(const ThresholdPreferenceFiles&) = delete; + ThresholdPreferenceFiles& operator=(ThresholdPreferenceFiles&&) = delete; +}; + +const std::array k_ThresholdKeys = {std::string(Preferences::k_LargeDataSize_Key), std::string(Preferences::k_LargeDataStructureSize_Key)}; +const std::array k_ThresholdLabels = {"Large Data Size", "Large Data Structure Size"}; + +std::vector invalidThresholdValues() +{ + return {nlohmann::json(int64{-1}), + nlohmann::json(1.25), + nlohmann::json(1.0), + nlohmann::json("0"), + nlohmann::json(true), + nlohmann::json(nullptr), + nlohmann::json::array({1}), + nlohmann::json::object({{"value", 1}}), + nlohmann::json(18446744073709551616.0)}; +} + +void writeThresholdJson(const std::filesystem::path& path, const nlohmann::json& value) +{ + std::ofstream stream(path); + REQUIRE(stream.is_open()); + stream << value; + stream.close(); + REQUIRE_FALSE(stream.fail()); +} + +nlohmann::json readThresholdJson(const std::filesystem::path& path) +{ + std::ifstream stream(path); + REQUIRE(stream.is_open()); + return nlohmann::json::parse(stream); +} +} // namespace + +TEST_CASE("Preferences rejects invalid new OOC thresholds before mutation", "[Core][Preferences][OocThreshold]") +{ + for(const auto& key : k_ThresholdKeys) + { + for(const auto& invalidValue : invalidThresholdValues()) + { + DYNAMIC_SECTION(key << "=" << invalidValue.dump()) + { + Preferences preferences; + preferences.setValue(key, uint64{17}); + preferences.setDataStorageMode(DataStorageMode::ForceInCore); + preferences.setPluginValue("ThresholdTest", "nested", nlohmann::json::object({{"keep", "unchanged"}})); + const auto originalDefault = preferences.defaultValue(Preferences::k_LargeDataStructureSize_Key); + + CHECK_THROWS_AS(preferences.setValue(key, invalidValue), std::invalid_argument); + CHECK(preferences.value(key) == nlohmann::json(uint64{17})); + CHECK(preferences.defaultValue(Preferences::k_LargeDataStructureSize_Key) == originalDefault); + CHECK(preferences.dataStorageMode() == DataStorageMode::ForceInCore); + CHECK(preferences.pluginValue("ThresholdTest", "nested") == nlohmann::json::object({{"keep", "unchanged"}})); + } + } + } +} + +TEST_CASE("Preferences preserves valid OOC threshold values", "[Core][Preferences][OocThreshold]") +{ + const std::vector validValues = {nlohmann::json(int64{0}), nlohmann::json(int64{37}), nlohmann::json(uint64{0}), nlohmann::json(uint64{37}), + nlohmann::json(std::numeric_limits::max())}; + for(const auto& key : k_ThresholdKeys) + { + for(const auto& validValue : validValues) + { + DYNAMIC_SECTION(key << "=" << validValue.dump() << " type=" << static_cast(validValue.type())) + { + ThresholdPreferenceFiles files; + Preferences preferences; + const auto originalDefault = preferences.defaultValue(Preferences::k_LargeDataStructureSize_Key); + REQUIRE_NOTHROW(preferences.setValue(key, validValue)); + CHECK(preferences.value(key) == validValue); + CHECK(preferences.value(key).type() == validValue.type()); + CHECK(preferences.defaultValue(Preferences::k_LargeDataStructureSize_Key) == originalDefault); + REQUIRE(preferences.saveToFile(files.input).valid()); + + // The file stores 0.01-GB steps (1 GB = 1024^3 bytes), so a load returns the nearest step. + // 37 bytes is above 0 and becomes 0.01 GB (10737418 bytes). UINT64_MAX becomes 17179869183.99 GB. + const std::string gbKey = key == Preferences::k_LargeDataSize_Key ? std::string(Preferences::k_LargeDataSizeGB_Key) : std::string(Preferences::k_LargeDataStructureSizeGB_Key); + const uint64 inputBytes = validValue.get(); + const uint64 expectedBytes = inputBytes == 0 ? uint64{0} : (inputBytes == 37 ? uint64{10737418} : uint64{18446744073698814198ULL}); + const double expectedGB = inputBytes == 0 ? 0.0 : (inputBytes == 37 ? 0.01 : 17179869183.99); + const auto written = readThresholdJson(files.input); + CHECK_FALSE(written.contains(key)); + REQUIRE(written.contains(gbKey)); + CHECK(written.at(gbKey).get() == expectedGB); + + Preferences loaded; + const auto result = loaded.loadFromFile(files.input); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + REQUIRE(loaded.contains(key)); + CHECK_FALSE(loaded.contains(gbKey)); + const auto loadedValue = loaded.value(key); + REQUIRE(loadedValue.is_number_unsigned()); + CHECK(loadedValue.get() == expectedBytes); + REQUIRE(loaded.saveToFile(files.output).valid()); + const auto saved = readThresholdJson(files.output); + CHECK_FALSE(saved.contains(key)); + REQUIRE(saved.contains(gbKey)); + CHECK(saved.at(gbKey).get() == expectedGB); + } + } + } +} + +TEST_CASE("Preferences preserves absent OOC threshold keys", "[Core][Preferences][OocThreshold]") +{ + ThresholdPreferenceFiles files; + writeThresholdJson(files.input, nlohmann::json::object({{"plugins", nlohmann::json::object()}})); + Preferences preferences; + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + REQUIRE(preferences.saveToFile(files.output).valid()); + const auto saved = readThresholdJson(files.output); + for(const auto& key : k_ThresholdKeys) + { + CHECK_FALSE(preferences.contains(key)); + CHECK(preferences.value(key) == preferences.defaultValue(key)); + CHECK_FALSE(saved.contains(key)); + } + CHECK_FALSE(saved.contains(Preferences::k_LargeDataSizeGB_Key)); + CHECK_FALSE(saved.contains(Preferences::k_LargeDataStructureSizeGB_Key)); +} + +TEST_CASE("Preferences repairs only invalid saved OOC threshold fields", "[Core][Preferences][OocThreshold]") +{ + for(usize keyIndex = 0; keyIndex < k_ThresholdKeys.size(); ++keyIndex) + { + const auto& invalidKey = k_ThresholdKeys[keyIndex]; + const auto& validKey = k_ThresholdKeys[1 - keyIndex]; + for(const auto& invalidValue : invalidThresholdValues()) + { + DYNAMIC_SECTION(invalidKey << "=" << invalidValue.dump()) + { + ThresholdPreferenceFiles files; + const auto nested = nlohmann::json::object({{"keep", nlohmann::json::array({"value", 43})}}); + const auto original = nlohmann::json::object({{invalidKey, invalidValue}, + {validKey, std::numeric_limits::max()}, + {"plugins", {{"ThresholdTest", nested}}}, + {"cache_memory_budget_bytes", uint64{123456}}, + {"unrelated", "preserved"}}); + writeThresholdJson(files.input, original); + + Preferences preferences; + const auto defaultBytes = preferences.defaultValueAs(invalidKey); + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 1); + const auto& message = result.warnings().front().message; + CHECK(message.find(k_ThresholdLabels[keyIndex]) != std::string::npos); + CHECK(message.find(invalidKey) != std::string::npos); + CHECK(message.find(invalidValue.dump()) != std::string::npos); + CHECK(message.find(invalidValue.type_name()) != std::string::npos); + CHECK(message.find(std::to_string(std::numeric_limits::max())) != std::string::npos); + CHECK(message.find(std::to_string(defaultBytes)) != std::string::npos); + CHECK(message.find("Preferences > Out of Core") != std::string::npos); + CHECK(result.warnings().front().code == -591); + CHECK(preferences.value(invalidKey) == preferences.defaultValue(invalidKey)); + REQUIRE(preferences.contains(validKey)); + const auto retainedValue = preferences.value(validKey); + REQUIRE(retainedValue.is_number_unsigned()); + // A legacy byte value loads as the nearest 0.01-GB step; UINT64_MAX becomes 17179869183.99 GB. + CHECK(retainedValue.get() == uint64{18446744073698814198ULL}); + CHECK(preferences.pluginValue("ThresholdTest", "keep") == nested.at("keep")); + CHECK(preferences.cacheMemoryBudgetBytes() == uint64{123456}); + CHECK(preferences.value("unrelated") == nlohmann::json("preserved")); + CHECK(readThresholdJson(files.input) == original); + } + } + } +} + +TEST_CASE("Preferences rejects non-object documents without changing existing state", "[Core][Preferences][OocThreshold]") +{ + const std::vector invalidRoots = {nlohmann::json::array({1}), nlohmann::json(nullptr), nlohmann::json("text"), nlohmann::json(7), nlohmann::json(true)}; + for(const auto& invalidRoot : invalidRoots) + { + DYNAMIC_SECTION(invalidRoot.dump()) + { + ThresholdPreferenceFiles files; + writeThresholdJson(files.input, invalidRoot); + Preferences preferences; + preferences.setValue(Preferences::k_LargeDataSize_Key, uint64{19}); + preferences.setValue(Preferences::k_LargeDataStructureSize_Key, std::numeric_limits::max()); + preferences.setDataStorageMode(DataStorageMode::ForceInCore); + preferences.setPluginValue("ThresholdTest", "keep", "unchanged"); + Result<> result; + CHECK_NOTHROW(result = preferences.loadFromFile(files.input)); + CHECK(result.invalid()); + CHECK(preferences.value(Preferences::k_LargeDataSize_Key) == nlohmann::json(uint64{19})); + CHECK(preferences.value(Preferences::k_LargeDataStructureSize_Key) == nlohmann::json(std::numeric_limits::max())); + CHECK(preferences.dataStorageMode() == DataStorageMode::ForceInCore); + CHECK(preferences.pluginValue("ThresholdTest", "keep") == nlohmann::json("unchanged")); + } + } +} + +TEST_CASE("Preferences validates OOC size values without mutation", "[Core][Preferences][OocThreshold]") +{ + for(const auto& key : k_ThresholdKeys) + { + for(const nlohmann::json value : {nlohmann::json(int64{0}), nlohmann::json(int64{17}), nlohmann::json(uint64{0}), nlohmann::json(uint64{17}), nlohmann::json(std::numeric_limits::max())}) + { + DYNAMIC_SECTION(key << "=" << value.dump() << " type=" << static_cast(value.type())) + { + const auto result = Preferences::ValidateOocSizeValue(key, value); + REQUIRE(result.valid()); + CHECK(result.value() == value.get()); + } + } + for(const auto& value : invalidThresholdValues()) + { + DYNAMIC_SECTION(key << "=" << value.dump() << " type=" << static_cast(value.type())) + { + const auto result = Preferences::ValidateOocSizeValue(key, value); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().message.find(key) != std::string::npos); + } + } + } + const auto unknown = Preferences::ValidateOocSizeValue("unrelated", nlohmann::json(uint64{17})); + REQUIRE(unknown.invalid()); + + Preferences preferences; + const nlohmann::json invalidUtf8(std::string(1, static_cast(0xff))); + REQUIRE_THROWS_AS(preferences.setValue(k_ThresholdKeys[0], invalidUtf8), std::invalid_argument); + CHECK_FALSE(preferences.contains(k_ThresholdKeys[0])); +} + +TEST_CASE("Preferences names nonfinite OOC thresholds without changing values", "[Core][Preferences][OocThreshold]") +{ + struct NonfiniteCase + { + double number; + const char* description; + }; + const std::array values = { + {{std::numeric_limits::quiet_NaN(), "NaN"}, {std::numeric_limits::infinity(), "+Infinity"}, {-std::numeric_limits::infinity(), "-Infinity"}}}; + for(const auto& key : k_ThresholdKeys) + { + for(const auto& nonfinite : values) + { + DYNAMIC_SECTION(key << "=" << nonfinite.description) + { + const nlohmann::json invalidValue(nonfinite.number); + const auto validation = Preferences::ValidateOocSizeValue(key, invalidValue); + REQUIRE(validation.invalid()); + REQUIRE(validation.errors().size() == 1); + CHECK(validation.errors().front().message.find(nonfinite.description) != std::string::npos); + + Preferences preferences; + preferences.setValue(key, uint64{17}); + try + { + preferences.setValue(key, invalidValue); + FAIL("A nonfinite threshold must be rejected."); + } catch(const std::invalid_argument& error) + { + CHECK(std::string(error.what()).find(nonfinite.description) != std::string::npos); + } + CHECK(preferences.value(key).is_number_unsigned()); + CHECK(preferences.valueAs(key) == uint64{17}); + } + } + } +} + +TEST_CASE("Preferences repairs both invalid saved OOC fields and reloads cleanly", "[Core][Preferences][OocThreshold]") +{ + ThresholdPreferenceFiles files; + const auto original = nlohmann::json::object({{k_ThresholdKeys[0], int64{-1}}, {k_ThresholdKeys[1], 1.25}, {"plugins", {{"ThresholdTest", {{"keep", "unchanged"}}}}}}); + writeThresholdJson(files.input, original); + Preferences preferences; + const auto expectedArray = preferences.defaultValueAs(k_ThresholdKeys[0]); + const auto expectedStructure = preferences.defaultValueAs(k_ThresholdKeys[1]); + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 2); + CHECK(preferences.valueAs(k_ThresholdKeys[0]) == expectedArray); + CHECK(preferences.valueAs(k_ThresholdKeys[1]) == expectedStructure); + CHECK(preferences.pluginValue("ThresholdTest", "keep") == nlohmann::json("unchanged")); + CHECK(readThresholdJson(files.input) == original); + REQUIRE(preferences.saveToFile(files.output).valid()); + Preferences reloaded; + const auto reload = reloaded.loadFromFile(files.output); + REQUIRE(reload.valid()); + CHECK(reload.warnings().empty()); + CHECK(reloaded.valueAs(k_ThresholdKeys[0]) == expectedArray); + CHECK(reloaded.valueAs(k_ThresholdKeys[1]) == expectedStructure); +} + +TEST_CASE("Preferences bounds invalid saved OOC value diagnostics", "[Core][Preferences][OocThreshold]") +{ + const std::array values = {nlohmann::json(std::string(4096, 'x')), nlohmann::json::object({{"nested", std::string(4096, 'y')}}), + nlohmann::json::array({"first-value", std::string(4096, 'z')})}; + for(const auto& invalidValue : values) + { + ThresholdPreferenceFiles files; + writeThresholdJson(files.input, nlohmann::json::object({{k_ThresholdKeys[0], invalidValue}})); + Preferences preferences; + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 1); + const auto& message = result.warnings().front().message; + CHECK(message.size() < 1024); + CHECK(message.find(k_ThresholdKeys[0]) != std::string::npos); + CHECK(message.find("Preferences > Out of Core") != std::string::npos); + if(invalidValue.is_object()) + { + CHECK(message.find("nested") != std::string::npos); + CHECK(message.find("yyyy") != std::string::npos); + } + if(invalidValue.is_array()) + { + CHECK(message.find("first-value") != std::string::npos); + } + } +} + +TEST_CASE("Preferences threshold repair preserves cache and storage migrations", "[Core][Preferences][OocThreshold]") +{ + ThresholdPreferenceFiles files; + writeThresholdJson(files.input, nlohmann::json::object({{std::string(Preferences::k_LargeDataSize_Key), int64{-1}}, + {"memory_budget_bytes", uint64{123456}}, + {std::string(Preferences::k_PreferredLargeDataFormat_Key), "In-Memory"}, + {"plugins", {{"ThresholdTest", {{"keep", "unchanged"}}}}}})); + + Preferences preferences; + const uint64 defaultBytes = preferences.defaultValueAs(Preferences::k_LargeDataSize_Key); + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 1); + CHECK(preferences.valueAs(Preferences::k_LargeDataSize_Key) == defaultBytes); + CHECK(preferences.cacheMemoryBudgetBytes() == uint64{123456}); + CHECK(preferences.contains(Preferences::k_CacheMemoryBudgetBytes_Key)); + CHECK_FALSE(preferences.contains(Preferences::k_LegacyMemoryBudgetBytes_Key)); + CHECK_FALSE(preferences.contains(Preferences::k_PreferredLargeDataFormat_Key)); + CHECK(preferences.dataStorageMode() == DataStorageMode::Adaptive); + CHECK(preferences.pluginValue("ThresholdTest", "keep") == nlohmann::json("unchanged")); +} + +TEST_CASE("Preferences syntax errors leave existing OOC values unchanged", "[Core][Preferences][OocThreshold]") +{ + ThresholdPreferenceFiles files; + std::ofstream stream(files.input); + REQUIRE(stream.is_open()); + stream << "{invalid"; + stream.close(); + + Preferences preferences; + preferences.setValue(Preferences::k_LargeDataSize_Key, std::numeric_limits::max()); + preferences.setDataStorageMode(DataStorageMode::ForceInCore); + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.invalid()); + const auto retained = preferences.value(Preferences::k_LargeDataSize_Key); + REQUIRE(retained.is_number_unsigned()); + CHECK(retained.get() == std::numeric_limits::max()); + CHECK(preferences.dataStorageMode() == DataStorageMode::ForceInCore); +} diff --git a/test/DataStoreBulkTest.cpp b/test/DataStoreBulkTest.cpp index 14c87bbbcd..a4cdd209a1 100644 --- a/test/DataStoreBulkTest.cpp +++ b/test/DataStoreBulkTest.cpp @@ -2,6 +2,7 @@ #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/EmptyDataStore.hpp" #include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" #include @@ -272,7 +273,7 @@ TEST_CASE("DataStore extent APIs preserve resident layouts and outside values", const Extent booleanExtent({0, 0, 0}, {0, 1, 3}, {1, 1, 2}); Result> booleanRead = booleanStore.readExtent(booleanExtent); REQUIRE(booleanRead.valid()); - REQUIRE(booleanRead.value() == std::vector{true, false, false, false, false, true, true, false}); + REQUIRE((booleanRead.value() == std::vector{true, false, false, false, false, true, true, false})); const Extent booleanContiguousExtent({0, 0, 0}, {0, 0, 1}); std::array booleanContiguousValues = {false, false, false, false}; @@ -488,3 +489,169 @@ TEST_CASE("DataArray deepCopy does not insert a destination after a source bulk- REQUIRE(dataStructure.getDataAs>(copyPath) == nullptr); REQUIRE(sourceStore->m_BulkReadCalls == 1); } + +TEST_CASE("DataStore bounded extent validates before writing", "[DataStore][bounded]") +{ + uint64 reviewedVectorBound = 0; + REQUIRE(BoundedRead::FreshVectorBytes(8, sizeof(int32), reviewedVectorBound)); + REQUIRE(reviewedVectorBound >= 8 * sizeof(int32)); + DataStore store(ShapeType{2, 3}, ShapeType{2}, 0); + for(usize value = 0; value < 12; ++value) + { + store.setValue(value, static_cast(value)); + } + const Extent extent({0, 0}, {1, 2}, {1, 2}); + std::vector destination(8, -99); + SECTION("subminimum allowance leaves the destination unchanged") + { + const auto result = store.readExtentIntoBufferBounded(extent, destination, BoundedRead::k_DiagnosticBytes - 1); + REQUIRE(result.valid()); + REQUIRE_FALSE(result.value()); + REQUIRE(result.warnings().empty()); + REQUIRE(destination == std::vector(8, -99)); + } + SECTION("strided complete tuples match the independent sequence") + { + const auto result = store.readExtentIntoBufferBounded(extent, destination, BoundedRead::k_DiagnosticBytes); + REQUIRE(result.valid()); + REQUIRE(result.value()); + REQUIRE((destination == std::vector{0, 1, 4, 5, 6, 7, 10, 11})); + } + SECTION("mismatched extent vectors are rejected before indexing") + { + Extent malformed = extent; + malformed.max.pop_back(); + const auto result = store.readExtentIntoBufferBounded(malformed, destination, BoundedRead::k_DiagnosticBytes); + REQUIRE(result.invalid()); + REQUIRE(destination == std::vector(8, -99)); + } + SECTION("short destination is unchanged on validation failure") + { + const auto result = store.readExtentIntoBufferBounded(extent, nonstd::span(destination.data(), 7), BoundedRead::k_DiagnosticBytes); + REQUIRE(result.invalid()); + REQUIRE(destination == std::vector(8, -99)); + } + SECTION("zero stride is rejected before division") + { + Extent malformed = extent; + malformed.stride[1] = 0; + const auto result = store.readExtentIntoBufferBounded(malformed, destination, BoundedRead::k_DiagnosticBytes); + REQUIRE(result.invalid()); + REQUIRE(destination == std::vector(8, -99)); + } +} + +TEST_CASE("Bounded extent arithmetic rejects shape and byte overflow without allocation", "[DataStore][bounded]") +{ + BoundedRead::Diagnostic diagnostic; + uint64 selected = 0; + const Extent extent({0, 0}, {0, 0}); + const std::array overflowingShape{std::numeric_limits::max(), 2}; + REQUIRE(BoundedRead::ValidateExtent(extent, overflowingShape, 1, 1, selected, diagnostic, sizeof(uint16)) == BoundedRead::Status::Failed); + REQUIRE(diagnostic.code == BoundedRead::InvalidExtent); + const Extent oneDimension({0}, {0}); + const std::array byteOverflow{std::numeric_limits::max()}; + REQUIRE(BoundedRead::ValidateExtent(oneDimension, byteOverflow, 1, 1, selected, diagnostic, sizeof(uint16)) == BoundedRead::Status::Failed); + REQUIRE(diagnostic.code == BoundedRead::InvalidExtent); +} + +TEST_CASE("Bounded diagnostics retain valid Unicode and explicitly escape malformed native units", "[DataStore][bounded]") +{ + const std::string supplementary = "\xF0\x9F\x8C\x90"; + BoundedRead::Diagnostic diagnostic; + diagnostic.set(BoundedRead::ReadFailure, "Read failed"); +#ifdef _WIN32 + const std::filesystem::path path(std::wstring{L'p', wchar_t{0xD83C}, wchar_t{0xDF10}, wchar_t{0xD800}}); + diagnostic.context(path, "data"); + const auto result = diagnostic.finish(BoundedRead::Status::Failed); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.errors().front().message.find(supplementary) != std::string::npos); + REQUIRE(result.errors().front().message.find("\\uD800") != std::string::npos); + REQUIRE(result.errors().front().message.find("malformed units escaped") != std::string::npos); +#else + const std::filesystem::path path(std::string("p") + supplementary + std::string(1, static_cast(0xFF))); + diagnostic.context(path, "data"); + const auto result = diagnostic.finish(BoundedRead::Status::Failed); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.errors().front().message.find(supplementary) != std::string::npos); + REQUIRE(result.errors().front().message.find("\\xFF") != std::string::npos); + REQUIRE(result.errors().front().message.find("invalid UTF-8/control bytes escaped") != std::string::npos); +#endif + REQUIRE(BoundedRead::ResultCapacityBytes(result) <= BoundedRead::k_DiagnosticBytes); + BoundedRead::Diagnostic truncated; + truncated.set(BoundedRead::ReadFailure, std::string(497, 'a')); + truncated.append(supplementary); + truncated.append(std::string(40, 'b')); + const auto truncatedResult = truncated.finish(BoundedRead::Status::Failed); + REQUIRE(truncatedResult.invalid()); + REQUIRE(truncatedResult.errors().size() == 1); + REQUIRE(truncatedResult.errors().front().message == std::string(497, 'a') + " [truncated]"); +} + +TEST_CASE("Default bounded store API declines without an allocating fallback", "[DataStore][bounded]") +{ + const auto created = EmptyDataStore::Create(ShapeType{1, 2}, ShapeType{1}, ""); + REQUIRE(created.valid()); + REQUIRE(created.value() != nullptr); + std::array destination{77, 88}; + const auto result = created.value()->readExtentIntoBufferBounded(Extent({0, 0}, {0, 1}), destination, BoundedRead::k_DiagnosticBytes); + REQUIRE(result.valid()); + REQUIRE_FALSE(result.value()); + REQUIRE(result.warnings().empty()); + REQUIRE(destination[0] == 77); + REQUIRE(destination[1] == 88); +} + +TEST_CASE("Bounded terminal representation requires reviewed native controls", "[DataStore][bounded]") +{ +#if defined(_MSVC_STL_VERSION) + INFO("MSVC STL release=" << _MSVC_STL_VERSION); +#if defined(_MSVC_STL_UPDATE) + INFO("MSVC STL update=" << _MSVC_STL_UPDATE); +#endif +#elif defined(_GLIBCXX_RELEASE) + INFO("libstdc++ release=" << _GLIBCXX_RELEASE); +#if defined(__GLIBCXX__) + INFO("libstdc++ date=" << __GLIBCXX__); +#endif +#if defined(_GLIBCXX_USE_CXX11_ABI) + INFO("libstdc++ CXX11 ABI=" << _GLIBCXX_USE_CXX11_ABI); +#endif +#elif defined(_LIBCPP_VERSION) + INFO("libc++ version=" << _LIBCPP_VERSION); +#if defined(_LIBCPP_ABI_VERSION) + INFO("libc++ ABI=" << _LIBCPP_ABI_VERSION); +#endif +#endif +#if defined(expected_lite_MAJOR) && defined(expected_lite_MINOR) && defined(expected_lite_PATCH) + INFO("expected-lite version=" << expected_lite_MAJOR << '.' << expected_lite_MINOR << '.' << expected_lite_PATCH); +#else + INFO("expected-lite version macros are unavailable"); +#endif +#if defined(nsel_USES_STD_EXPECTED) + INFO("nsel_USES_STD_EXPECTED=" << nsel_USES_STD_EXPECTED); +#endif + REQUIRE(BoundedRead::DiagnosticRepresentationSupported()); + REQUIRE(BoundedRead::TerminalDiagnosticBoundBytes() > 0); + REQUIRE(BoundedRead::TerminalDiagnosticBoundBytes() <= BoundedRead::k_DiagnosticBytes); + // These scopes deliberately keep the named return and its moved-from control alive. + // The supported Debug configuration retains its normal empty-vector bookkeeping. + BoundedRead::Diagnostic diagnostic; + diagnostic.counts(BoundedRead::Insufficient, "Insufficient test allowance", 8192, 4096); + auto named = diagnostic.finish(BoundedRead::Status::Unavailable); + REQUIRE(named.valid()); + REQUIRE_FALSE(named.value()); + REQUIRE(named.warnings().size() == 1); + const auto message = named.warnings().front().message; + auto received = std::move(named); + REQUIRE(received.warnings().front().message == message); + REQUIRE(BoundedRead::ResultCapacityBytes(received) <= BoundedRead::TerminalDiagnosticBoundBytes()); + diagnostic.counts(BoundedRead::InvalidExtent, "Invalid test count", 7, 6); + auto failure = diagnostic.finish(BoundedRead::Status::Failed); + REQUIRE(failure.invalid()); + REQUIRE(failure.errors().size() == 1); + REQUIRE(failure.errors().front().message.find("required=7, available=6") != std::string::npos); + REQUIRE(BoundedRead::ResultCapacityBytes(failure) <= BoundedRead::TerminalDiagnosticBoundBytes()); +} diff --git a/test/DataStructTest.cpp b/test/DataStructTest.cpp index bc34de8b64..2bfdc79e64 100644 --- a/test/DataStructTest.cpp +++ b/test/DataStructTest.cpp @@ -1,6 +1,7 @@ #include "DataStructObserver.hpp" #include "simplnx/Common/StringLiteral.hpp" +#include "simplnx/DataStructure/AttributeMatrix.hpp" #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataGroup.hpp" #include "simplnx/DataStructure/DataStore.hpp" @@ -12,6 +13,8 @@ #include "simplnx/DataStructure/Geometry/RectGridGeom.hpp" #include "simplnx/DataStructure/Geometry/TetrahedralGeom.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" +#include "simplnx/DataStructure/Messaging/DataAddedMessage.hpp" +#include "simplnx/DataStructure/Messaging/DataRemovedMessage.hpp" #include "simplnx/DataStructure/ScalarData.hpp" #include "simplnx/DataStructure/StringArray.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" @@ -20,10 +23,15 @@ #include "simplnx/unit_test/simplnx_test_dirs.hpp" #include +#include +#include #include #include #include +#include +#include +#include #include namespace fs = std::filesystem; @@ -1675,3 +1683,890 @@ TEST_CASE("CompareDataStructures-fail", "[SimplnxReview][!mayfail]") UnitTest::CompareDataStructures(dataStructure0, dataStructure1); } + +namespace +{ +/** + * @brief Creates a small resident fixture with explicit IDs and literal values. + * @param structure Owns the array. + * @param name Identifies the root array. + * @param objectId Supplies the fixture ID. + * @param values Supplies the independent expected values. + * @return The exact inserted array owner. + */ +std::shared_ptr createImportPublicationArray(DataStructure& structure, const std::string& name, DataObject::IdType objectId, const std::array& values) +{ + auto store = std::make_shared>(ShapeType{2}, ShapeType{1}, 0); + (*store)[0] = values[0]; + (*store)[1] = values[1]; + REQUIRE(Int32Array::Import(structure, name, objectId, store) != nullptr); + auto owner = std::dynamic_pointer_cast(structure.getSharedData(objectId)); + REQUIRE(owner != nullptr); + return owner; +} + +/** + * @class ImportCopyFailureGroup + * @brief Throws during detached-copy construction to exercise association restoration. + */ +class ImportCopyFailureGroup : public DataGroup +{ +public: + /** + * @brief Creates a group whose shallow copy fails. + * @param structure Supplies the source association. + */ + explicit ImportCopyFailureGroup(DataStructure& structure) + : DataGroup(structure, "CopyFailure") + { + } + + ~ImportCopyFailureGroup() noexcept override = default; + ImportCopyFailureGroup(const ImportCopyFailureGroup&) = default; + ImportCopyFailureGroup(ImportCopyFailureGroup&&) = delete; + ImportCopyFailureGroup& operator=(const ImportCopyFailureGroup&) = delete; + ImportCopyFailureGroup& operator=(ImportCopyFailureGroup&&) = delete; + + /** + * @brief Throws before a new owner is constructed. + * @return Does not return. + * @throws std::runtime_error Always, as the fixture's copy failure. + */ + DataObject* shallowCopy() override + { + throw std::runtime_error("import detached-copy failure witness"); + } +}; +} // namespace + +TEST_CASE("Import publication: group acceptance queries preserve state and virtual rules", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + DataStructure destination; + auto* parentPtr = DataGroup::Create(destination, "Parent"); + REQUIRE(parentPtr != nullptr); + auto* childPtr = DataGroup::Create(destination, "Child", parentPtr->getId()); + REQUIRE(childPtr != nullptr); + auto* matrixPtr = AttributeMatrix::Create(destination, "Matrix", ShapeType{2}); + REQUIRE(matrixPtr != nullptr); + DataStructure candidates; + auto matching = createImportPublicationArray(candidates, "Matching", 10, std::array{31, -7}); + auto mismatchStore = std::make_shared>(ShapeType{3}, ShapeType{1}, 0); + auto* mismatchPtr = Int32Array::Create(candidates, "Mismatch", mismatchStore); + REQUIRE(mismatchPtr != nullptr); + auto* duplicateNamePtr = DataGroup::Create(candidates, "Child"); + REQUIRE(duplicateNamePtr != nullptr); + const auto paths = destination.getAllDataPaths(); + const auto ids = destination.getAllDataObjectIds(); + const auto nextId = destination.getNextId(); + const auto parents = childPtr->getParentIds(); + const auto candidatePaths = candidates.getAllDataPaths(); + const auto candidateNextId = candidates.getNextId(); + usize notifications = 0; + const nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const BaseGroup& parentView = *parentPtr; + const BaseGroup& matrixView = *matrixPtr; + CHECK_FALSE(parentView.acceptsChild(nullptr)); + CHECK_FALSE(parentView.acceptsChild(childPtr)); + CHECK_FALSE(parentView.acceptsChild(duplicateNamePtr)); + CHECK_FALSE(childPtr->acceptsChild(parentPtr)); + CHECK(parentView.acceptsChild(matching.get())); + CHECK_FALSE(matrixView.acceptsChild(parentPtr)); + CHECK_FALSE(matrixView.acceptsChild(mismatchPtr)); + CHECK(matrixView.acceptsChild(matching.get())); + CHECK(destination.getAllDataPaths() == paths); + CHECK(destination.getAllDataObjectIds() == ids); + CHECK(destination.getNextId() == nextId); + CHECK(childPtr->getParentIds() == parents); + CHECK(candidates.getAllDataPaths() == candidatePaths); + CHECK(candidates.getNextId() == candidateNextId); + CHECK(matching->getParentIds().empty()); + CHECK(notifications == 0); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(candidates); +} + +TEST_CASE("Import publication: occupied IDs include expired entries and hierarchy-only aliases", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + DataStructure destination; + auto* markerPtr = DataGroup::Create(destination, "Marker"); + REQUIRE(markerPtr != nullptr); + auto* expiredPtr = DataGroup::Create(destination, "Expired"); + REQUIRE(expiredPtr != nullptr); + REQUIRE(markerPtr->getId() == 1); + REQUIRE(expiredPtr->getId() == 2); + REQUIRE(destination.removeData(DataObject::IdType{2})); + REQUIRE(destination.getData(DataObject::IdType{2}) == nullptr); + DataStructure prepared; + auto* sourcePtr = DataGroup::Import(prepared, "HierarchyOnly", 7); + REQUIRE(sourcePtr != nullptr); + auto shell = DataStructure::makeImportPublicationCopy(*sourcePtr); + REQUIRE(shell != nullptr); + // Raw hierarchy placement models the same unregistered stage as a partial import. + REQUIRE(destination.getRootGroup().insert(shell)); + REQUIRE(markerPtr->getDataMap().insert(shell)); + CHECK(destination.getData(DataObject::IdType{7}) == nullptr); + const auto nextId = destination.getNextId(); + CHECK(destination.getLiveObjectCount() == 1); + CHECK((destination.getOccupiedObjectIds() == std::set{1, 2, 7})); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2})); + CHECK(destination.getNextId() == nextId); + const auto rootPlacement = destination.getRootGroup().find(7); + REQUIRE(rootPlacement != destination.getRootGroup().end()); + CHECK(rootPlacement->second == shell); + const auto childPlacement = markerPtr->getDataMap().find(7); + REQUIRE(childPlacement != markerPtr->getDataMap().end()); + CHECK(childPlacement->second == shell); + REQUIRE(markerPtr->getDataMap().remove(DataObject::IdType{7})); + REQUIRE(destination.getRootGroup().remove(DataObject::IdType{7})); + CHECK(shell->getDataStructure() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("Import publication: detached copies preserve the source and share only value storage", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + DataStructure source; + auto* parentPtr = DataGroup::Create(source, "Parent"); + REQUIRE(parentPtr != nullptr); + auto store = std::make_shared>(ShapeType{2}, ShapeType{1}, 0); + (*store)[0] = 31; + (*store)[1] = -7; + auto* arrayPtr = Int32Array::Create(source, "Values", store, parentPtr->getId()); + REQUIRE(arrayPtr != nullptr); + const auto arrayId = arrayPtr->getId(); + const auto parentIds = arrayPtr->getParentIds(); + const auto paths = source.getAllDataPaths(); + const auto nextId = source.getNextId(); + usize notifications = 0; + const nod::scoped_connection connection(source.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + { + const auto groupCopy = std::dynamic_pointer_cast(DataStructure::makeImportPublicationCopy(*parentPtr)); + REQUIRE(groupCopy != nullptr); + CHECK(groupCopy.get() != parentPtr); + CHECK(groupCopy->getId() == parentPtr->getId()); + CHECK(groupCopy->empty()); + CHECK(groupCopy->getDataStructure() == nullptr); + CHECK(groupCopy->getParentIds().empty()); + const auto arrayCopy = std::dynamic_pointer_cast(DataStructure::makeImportPublicationCopy(*arrayPtr)); + REQUIRE(arrayCopy != nullptr); + CHECK(arrayCopy.get() != arrayPtr); + CHECK(arrayCopy->getId() == arrayId); + CHECK(arrayCopy->getDataStructure() == nullptr); + CHECK(arrayCopy->getParentIds().empty()); + CHECK(arrayCopy->getIDataStore() == store.get()); + CHECK((*arrayCopy)[0] == 31); + CHECK((*arrayCopy)[1] == -7); + } + CHECK(parentPtr->getDataStructure() == &source); + CHECK(arrayPtr->getDataStructure() == &source); + CHECK(arrayPtr->getParentIds() == parentIds); + CHECK(parentPtr->getSize() == 1); + CHECK(source.getData(arrayId) == arrayPtr); + CHECK(source.getAllDataPaths() == paths); + CHECK(source.getNextId() == nextId); + CHECK(notifications == 0); + auto failing = std::make_shared(source); + REQUIRE(source.insert(failing, DataPath{})); + const auto failureId = failing->getId(); + const auto failurePaths = source.getAllDataPaths(); + const auto failureNextId = source.getNextId(); + const auto beforeFailure = notifications; + REQUIRE_THROWS_AS(DataStructure::makeImportPublicationCopy(*failing), std::runtime_error); + CHECK(failing->getDataStructure() == &source); + CHECK(source.getData(failureId) == failing.get()); + CHECK(source.getAllDataPaths() == failurePaths); + CHECK(source.getNextId() == failureNextId); + CHECK(notifications == beforeFailure); + UnitTest::CheckArraysInheritTupleDims(source); +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + +TEST_CASE("Import publication: existing root names reject the complete selection before publication", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + CAPTURE(instrumented); + DataStructure destination; + auto marker = createImportPublicationArray(destination, "Marker", 1, std::array{51, 52}); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto candidate = createImportPublicationArray(prepared, "Candidate", 2, std::array{31, -7}); + auto conflict = createImportPublicationArray(prepared, "Marker", 3, std::array{81, 82}); + const auto* markerStorePtr = marker->getIDataStore(); + const auto* candidateStorePtr = candidate->getIDataStore(); + const auto* conflictStorePtr = conflict->getIDataStore(); + const std::vector selected{DataPath({"Candidate"}), DataPath({"Marker"})}; + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, selected, fs::path("root-name-conflict.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, selected, fs::path("root-name-conflict.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6203); + CHECK(result.errors().front().message.find("'Marker'") != std::string::npos); + CHECK(result.errors().front().message.find("root-name-conflict.dream3d") != std::string::npos); + CHECK(result.errors().front().message.find("destination path already exists") != std::string::npos); + CHECK(result.warnings().empty()); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"})})); + CHECK(destination.getData(DataObject::IdType{1}) == marker.get()); + CHECK(destination.getData(DataPath({"Marker"})) == marker.get()); + CHECK(destination.getData(DataObject::IdType{2}) == nullptr); + CHECK(destination.getData(DataObject::IdType{3}) == nullptr); + CHECK(destination.getNextId() == 2); + CHECK(marker->getIDataStore() == markerStorePtr); + CHECK((*marker)[0] == 51); + CHECK((*marker)[1] == 52); + CHECK((prepared.getAllDataObjectIds() == std::vector{2, 3})); + CHECK((prepared.getAllDataPaths() == std::vector{DataPath({"Candidate"}), DataPath({"Marker"})})); + CHECK(prepared.getData(DataObject::IdType{2}) == candidate.get()); + CHECK(prepared.getData(DataObject::IdType{3}) == conflict.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK(conflict->getDataStructure() == &prepared); + CHECK(candidate->getParentIds().empty()); + CHECK(conflict->getParentIds().empty()); + CHECK(candidate->getIDataStore() == candidateStorePtr); + CHECK(conflict->getIDataStore() == conflictStorePtr); + CHECK((*candidate)[0] == 31); + CHECK((*candidate)[1] == -7); + CHECK((*conflict)[0] == 81); + CHECK((*conflict)[1] == 82); + CHECK(prepared.getNextId() == 4); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + + if(instrumented) + { + const auto fault = control.publish(destination, prepared, {DataPath({"Candidate"})}, fs::path("root-name-conflict.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(fault); + REQUIRE(fault.errors().size() == 1); + CHECK(fault.errors().front().code == -6214); + REQUIRE(control.publicationFaultConsumed()); + CHECK(destination.getNextId() == 2); + CHECK(destination.getData(DataObject::IdType{2}) == nullptr); + CHECK(notifications == 0); + } + const auto retry = instrumented ? control.publish(destination, prepared, {DataPath({"Candidate"})}, fs::path("root-name-conflict.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, {DataPath({"Candidate"})}, fs::path("root-name-conflict.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_VALID(retry); + auto published = std::dynamic_pointer_cast(destination.getSharedData(DataObject::IdType{2})); + REQUIRE(published != nullptr); + CHECK(published.get() != candidate.get()); + CHECK(published->getIDataStore() == candidateStorePtr); + CHECK((*published)[0] == 31); + CHECK((*published)[1] == -7); + CHECK(destination.getData(DataPath({"Marker"})) == marker.get()); + CHECK(destination.getData(DataPath({"Candidate"})) == published.get()); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2})); + CHECK(destination.getNextId() == 3); + CHECK(prepared.getData(DataObject::IdType{2}) == candidate.get()); + CHECK(prepared.getData(DataObject::IdType{3}) == conflict.get()); + CHECK(prepared.getNextId() == 4); + CHECK(notifications == 0); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: hierarchy-only IDs in another parent reject before publication", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + CAPTURE(instrumented); + DataStructure destination; + auto* parentPtr = DataGroup::Import(destination, "Parent", 1); + REQUIRE(parentPtr != nullptr); + DataStructure source; + auto* sourcePtr = DataGroup::Import(source, "HierarchyOnly", 7); + REQUIRE(sourcePtr != nullptr); + auto occupied = DataStructure::makeImportPublicationCopy(*sourcePtr); + REQUIRE(occupied != nullptr); + // The child map owns ID 7, but the weak registry and the destination root do not. + REQUIRE(parentPtr->getDataMap().insert(occupied)); + REQUIRE(destination.getData(DataObject::IdType{7}) == nullptr); + REQUIRE(destination.getRootGroup().find(7) == destination.getRootGroup().end()); + REQUIRE(destination.getData(DataPath({"Parent", "HierarchyOnly"})) == occupied.get()); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto candidate = createImportPublicationArray(prepared, "Candidate", 7, std::array{31, -7}); + const auto* candidateStorePtr = candidate->getIDataStore(); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, {DataPath({"Candidate"})}, fs::path("hierarchy-id-conflict.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, {DataPath({"Candidate"})}, fs::path("hierarchy-id-conflict.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6213); + CHECK(result.errors().front().message.find("'Candidate'") != std::string::npos); + CHECK(result.errors().front().message.find("identifier 7") != std::string::npos); + CHECK(result.errors().front().message.find("hierarchy-id-conflict.dream3d") != std::string::npos); + CHECK(result.warnings().empty()); + CHECK(destination.getData(DataObject::IdType{1}) == parentPtr); + CHECK(destination.getData(DataObject::IdType{7}) == nullptr); + CHECK(destination.getData(DataPath({"Parent"})) == parentPtr); + CHECK(destination.getData(DataPath({"Parent", "HierarchyOnly"})) == occupied.get()); + CHECK(destination.getData(DataPath({"Candidate"})) == nullptr); + CHECK(destination.getRootGroup().getSize() == 1); + CHECK(parentPtr->getSize() == 1); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Parent"})})); + CHECK((destination.getOccupiedObjectIds() == std::set{1, 7})); + CHECK(destination.getNextId() == 2); + CHECK(occupied->getId() == 7); + CHECK(occupied->getDataStructure() == nullptr); + CHECK(occupied->getParentIds().empty()); + CHECK((prepared.getAllDataObjectIds() == std::vector{7})); + CHECK((prepared.getAllDataPaths() == std::vector{DataPath({"Candidate"})})); + CHECK(prepared.getData(DataObject::IdType{7}) == candidate.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK(candidate->getParentIds().empty()); + CHECK(candidate->getIDataStore() == candidateStorePtr); + CHECK((*candidate)[0] == 31); + CHECK((*candidate)[1] == -7); + CHECK(prepared.getNextId() == 8); + CHECK(source.getData(DataObject::IdType{7}) == sourcePtr); + CHECK(sourcePtr->getDataStructure() == &source); + CHECK(source.getNextId() == 8); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + REQUIRE(parentPtr->getDataMap().remove(DataObject::IdType{7})); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: duplicate selected paths reject before publication", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + CAPTURE(instrumented); + DataStructure destination; + auto* markerPtr = DataGroup::Import(destination, "Marker", 1); + REQUIRE(markerPtr != nullptr); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto candidate = createImportPublicationArray(prepared, "Candidate", 2, std::array{31, -7}); + const auto* storePtr = candidate->getIDataStore(); + const std::vector selected{DataPath({"Candidate"}), DataPath({"Candidate"})}; + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, selected, fs::path("duplicate-path.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, selected, fs::path("duplicate-path.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6203); + CHECK(result.errors().front().message.find("'Candidate'") != std::string::npos); + CHECK(result.errors().front().message.find("duplicate-path.dream3d") != std::string::npos); + CHECK(result.errors().front().message.find("same path is selected more than once") != std::string::npos); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"})})); + CHECK(destination.getData(DataObject::IdType{1}) == markerPtr); + CHECK(destination.getData(DataObject::IdType{2}) == nullptr); + CHECK(destination.getNextId() == 2); + CHECK((prepared.getAllDataObjectIds() == std::vector{2})); + CHECK((prepared.getAllDataPaths() == std::vector{DataPath({"Candidate"})})); + CHECK(prepared.getData(DataObject::IdType{2}) == candidate.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK(candidate->getParentIds().empty()); + CHECK(candidate->getIDataStore() == storePtr); + CHECK((*candidate)[0] == 31); + CHECK((*candidate)[1] == -7); + CHECK(prepared.getNextId() == 3); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: distinct selected alias paths cannot publish the same ID twice", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + CAPTURE(instrumented); + DataStructure destination; + auto* leftPtr = DataGroup::Import(destination, "Left", 1); + auto* rightPtr = DataGroup::Import(destination, "Right", 2); + REQUIRE(leftPtr != nullptr); + REQUIRE(rightPtr != nullptr); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto* preparedLeftPtr = DataGroup::Import(prepared, "Left", 1); + auto* preparedRightPtr = DataGroup::Import(prepared, "Right", 2); + REQUIRE(preparedLeftPtr != nullptr); + REQUIRE(preparedRightPtr != nullptr); + auto store = std::make_shared>(ShapeType{2}, ShapeType{1}, 0); + (*store)[0] = 31; + (*store)[1] = -7; + REQUIRE(Int32Array::Import(prepared, "Candidate", 3, store, DataObject::IdType{1}) != nullptr); + auto candidate = std::dynamic_pointer_cast(prepared.getSharedData(DataObject::IdType{3})); + REQUIRE(candidate != nullptr); + REQUIRE(prepared.setAdditionalParent(3, 2)); + const std::vector selected{DataPath({"Left", "Candidate"}), DataPath({"Right", "Candidate"})}; + REQUIRE(prepared.getData(selected[0]) == candidate.get()); + REQUIRE(prepared.getData(selected[1]) == candidate.get()); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, selected, fs::path("duplicate-id.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, selected, fs::path("duplicate-id.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6213); + CHECK(result.errors().front().message.find("Right/Candidate") != std::string::npos); + CHECK(result.errors().front().message.find("identifier 3") != std::string::npos); + CHECK(result.errors().front().message.find("duplicate-id.dream3d") != std::string::npos); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Left"}), DataPath({"Right"})})); + CHECK(destination.getData(DataObject::IdType{1}) == leftPtr); + CHECK(destination.getData(DataObject::IdType{2}) == rightPtr); + CHECK(destination.getData(DataObject::IdType{3}) == nullptr); + CHECK(leftPtr->empty()); + CHECK(rightPtr->empty()); + CHECK(destination.getNextId() == 3); + CHECK((prepared.getAllDataObjectIds() == std::vector{1, 2, 3})); + CHECK((prepared.getAllDataPaths() == std::vector{DataPath({"Left"}), DataPath({"Right"}), DataPath({"Left", "Candidate"}), DataPath({"Right", "Candidate"})})); + CHECK(prepared.getData(DataObject::IdType{1}) == preparedLeftPtr); + CHECK(prepared.getData(DataObject::IdType{2}) == preparedRightPtr); + CHECK(prepared.getData(DataObject::IdType{3}) == candidate.get()); + CHECK(prepared.getData(selected[0]) == candidate.get()); + CHECK(prepared.getData(selected[1]) == candidate.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK((candidate->getParentIds() == DataObject::ParentCollectionType{1, 2})); + CHECK(candidate->getIDataStore() == store.get()); + CHECK((*candidate)[0] == 31); + CHECK((*candidate)[1] == -7); + CHECK(prepared.getNextId() == 4); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: distinct parent aliases reject duplicate destination placements", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + CAPTURE(instrumented); + DataStructure destination; + auto* leftPtr = DataGroup::Import(destination, "Left", 1); + auto* rightPtr = DataGroup::Import(destination, "Right", 2); + REQUIRE(leftPtr != nullptr); + REQUIRE(rightPtr != nullptr); + auto* sharedParentPtr = DataGroup::Import(destination, "Shared", 3, DataObject::IdType{1}); + REQUIRE(sharedParentPtr != nullptr); + REQUIRE(destination.setAdditionalParent(3, 2)); + REQUIRE(destination.getData(DataPath({"Left", "Shared"})) == sharedParentPtr); + REQUIRE(destination.getData(DataPath({"Right", "Shared"})) == sharedParentPtr); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto* preparedLeftPtr = DataGroup::Import(prepared, "Left", 1); + auto* preparedRightPtr = DataGroup::Import(prepared, "Right", 2); + REQUIRE(preparedLeftPtr != nullptr); + REQUIRE(preparedRightPtr != nullptr); + auto* firstParentPtr = DataGroup::Import(prepared, "Shared", 3, DataObject::IdType{1}); + auto* secondParentPtr = DataGroup::Import(prepared, "Shared", 4, DataObject::IdType{2}); + REQUIRE(firstParentPtr != nullptr); + REQUIRE(secondParentPtr != nullptr); + REQUIRE(firstParentPtr != secondParentPtr); + auto firstStore = std::make_shared>(ShapeType{2}, ShapeType{1}, 0); + (*firstStore)[0] = 31; + (*firstStore)[1] = -7; + auto secondStore = std::make_shared>(ShapeType{2}, ShapeType{1}, 0); + (*secondStore)[0] = 81; + (*secondStore)[1] = 82; + auto* firstPtr = Int32Array::Import(prepared, "Candidate", 5, firstStore, DataObject::IdType{3}); + auto* secondPtr = Int32Array::Import(prepared, "Candidate", 6, secondStore, DataObject::IdType{4}); + REQUIRE(firstPtr != nullptr); + REQUIRE(secondPtr != nullptr); + const std::vector selected{DataPath({"Left", "Shared", "Candidate"}), DataPath({"Right", "Shared", "Candidate"})}; + REQUIRE(prepared.getData(selected[0]) == firstPtr); + REQUIRE(prepared.getData(selected[1]) == secondPtr); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, selected, fs::path("aliased-parent.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, selected, fs::path("aliased-parent.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6203); + CHECK(result.errors().front().message.find("Right/Shared/Candidate") != std::string::npos); + CHECK(result.errors().front().message.find("aliased-parent.dream3d") != std::string::npos); + CHECK(result.errors().front().message.find("destination placement is selected more than once") != std::string::npos); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2, 3})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Left"}), DataPath({"Right"}), DataPath({"Left", "Shared"}), DataPath({"Right", "Shared"})})); + CHECK(destination.getData(DataObject::IdType{1}) == leftPtr); + CHECK(destination.getData(DataObject::IdType{2}) == rightPtr); + CHECK(destination.getData(DataObject::IdType{3}) == sharedParentPtr); + CHECK(destination.getData(DataObject::IdType{5}) == nullptr); + CHECK(destination.getData(DataObject::IdType{6}) == nullptr); + CHECK(destination.getData(selected[0]) == nullptr); + CHECK(destination.getData(selected[1]) == nullptr); + CHECK(sharedParentPtr->empty()); + CHECK((sharedParentPtr->getParentIds() == DataObject::ParentCollectionType{1, 2})); + CHECK(destination.getNextId() == 4); + CHECK((prepared.getAllDataObjectIds() == std::vector{1, 2, 3, 4, 5, 6})); + CHECK((prepared.getAllDataPaths() == std::vector{DataPath({"Left"}), DataPath({"Right"}), DataPath({"Left", "Shared"}), DataPath({"Right", "Shared"}), + DataPath({"Left", "Shared", "Candidate"}), DataPath({"Right", "Shared", "Candidate"})})); + CHECK(prepared.getData(DataObject::IdType{1}) == preparedLeftPtr); + CHECK(prepared.getData(DataObject::IdType{2}) == preparedRightPtr); + CHECK(prepared.getData(DataObject::IdType{3}) == firstParentPtr); + CHECK(prepared.getData(DataObject::IdType{4}) == secondParentPtr); + CHECK(prepared.getData(DataObject::IdType{5}) == firstPtr); + CHECK(prepared.getData(DataObject::IdType{6}) == secondPtr); + CHECK(prepared.getData(selected[0]) == firstPtr); + CHECK(prepared.getData(selected[1]) == secondPtr); + CHECK(firstPtr->getDataStructure() == &prepared); + CHECK(secondPtr->getDataStructure() == &prepared); + CHECK((firstPtr->getParentIds() == DataObject::ParentCollectionType{3})); + CHECK((secondPtr->getParentIds() == DataObject::ParentCollectionType{4})); + CHECK(firstPtr->getIDataStore() == firstStore.get()); + CHECK(secondPtr->getIDataStore() == secondStore.get()); + CHECK((*firstPtr)[0] == 31); + CHECK((*firstPtr)[1] == -7); + CHECK((*secondPtr)[0] == 81); + CHECK((*secondPtr)[1] == 82); + CHECK(prepared.getNextId() == 7); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: checkpoint errors precede conflicting selection admission", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool foreign = GENERATE(false, true); + const bool instrumented = GENERATE(false, true); + CAPTURE(foreign, instrumented); + const DataStructure original; + DataStructure destination; + auto* markerPtr = DataGroup::Import(destination, "Marker", 1); + REQUIRE(markerPtr != nullptr); + const DataStructure::ImportPublicationCheckpoint checkpoint(foreign ? original : destination); + auto* laterPtr = DataGroup::Import(destination, "Later", 2); + REQUIRE(laterPtr != nullptr); + DataStructure prepared; + auto* conflictPtr = DataGroup::Import(prepared, "Marker", 1); + REQUIRE(conflictPtr != nullptr); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, {DataPath({"Marker"}), DataPath({"Marker"})}, fs::path("checkpoint-priority.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, {DataPath({"Marker"}), DataPath({"Marker"})}, fs::path("checkpoint-priority.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == (foreign ? -6217 : -6218)); + CHECK(destination.getData(DataObject::IdType{1}) == markerPtr); + CHECK(destination.getData(DataObject::IdType{2}) == laterPtr); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"}), DataPath({"Later"})})); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2})); + CHECK(destination.getNextId() == 3); + CHECK(prepared.getData(DataObject::IdType{1}) == conflictPtr); + CHECK(prepared.getNextId() == 2); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: empty and fully recovered selections preserve pending faults", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool empty = GENERATE(false, true); + const bool instrumented = GENERATE(false, true); + CAPTURE(empty, instrumented); + DataStructure destination; + auto* markerPtr = DataGroup::Import(destination, "Marker", 1); + REQUIRE(markerPtr != nullptr); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + const std::vector selected = empty ? std::vector{} : std::vector{DataPath({"Missing"}), DataPath({"Marker"}), DataPath({"Missing"})}; + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, selected, fs::path("empty-recovery.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, selected, fs::path("empty-recovery.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.warnings().empty()); + CHECK(destination.getData(DataObject::IdType{1}) == markerPtr); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"})})); + CHECK(destination.getNextId() == 2); + CHECK(prepared.getAllDataObjectIds().empty()); + CHECK(prepared.getAllDataPaths().empty()); + CHECK(prepared.getNextId() == 1); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: missing recovered paths are skipped before admission", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + CAPTURE(instrumented); + DataStructure destination; + auto* markerPtr = DataGroup::Import(destination, "Marker", 1); + REQUIRE(markerPtr != nullptr); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto candidate = createImportPublicationArray(prepared, "Candidate", 2, std::array{31, -7}); + const std::vector selected{DataPath({"Missing"}), DataPath({"Marker"}), DataPath({"Missing"}), DataPath({"Candidate"})}; + DataStructure::ImportPublicationTestControl control; + usize notifications = 0; + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection preparedConnection(prepared.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const auto result = instrumented ? control.publish(destination, prepared, selected, fs::path("recovered-paths.dream3d"), checkpoint) : + destination.publishImportedObjects(prepared, selected, fs::path("recovered-paths.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.warnings().empty()); + auto published = std::dynamic_pointer_cast(destination.getSharedData(DataObject::IdType{2})); + REQUIRE(published != nullptr); + CHECK(published.get() != candidate.get()); + CHECK(published->getIDataStore() == candidate->getIDataStore()); + CHECK((*published)[0] == 31); + CHECK((*published)[1] == -7); + CHECK(destination.getData(DataObject::IdType{1}) == markerPtr); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"}), DataPath({"Candidate"})})); + CHECK(destination.getNextId() == 3); + CHECK(prepared.getData(DataObject::IdType{2}) == candidate.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK(candidate->getParentIds().empty()); + CHECK((prepared.getAllDataObjectIds() == std::vector{2})); + CHECK((prepared.getAllDataPaths() == std::vector{DataPath({"Candidate"})})); + CHECK(prepared.getNextId() == 3); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: foreign checkpoints reject before fault consumption", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool instrumented = GENERATE(false, true); + DataStructure originalDestination; + auto originalMarker = createImportPublicationArray(originalDestination, "OriginalMarker", 1, std::array{41, 42}); + const DataStructure::ImportPublicationCheckpoint checkpoint(originalDestination); + DataStructure destination; + auto marker = createImportPublicationArray(destination, "Marker", 1, std::array{51, 52}); + const auto* markerStorePtr = marker->getIDataStore(); + DataStructure prepared; + auto candidate = createImportPublicationArray(prepared, "Candidate", 2, std::array{31, -7}); + const auto* candidateStorePtr = candidate->getIDataStore(); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection originalConnection(originalDestination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + const nod::scoped_connection destinationConnection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + Result<> result; + if(instrumented) + { + result = control.publish(destination, prepared, {DataPath({"Candidate"})}, fs::path("foreign-checkpoint.dream3d"), checkpoint); + } + else + { + result = destination.publishImportedObjects(prepared, {DataPath({"Candidate"})}, fs::path("foreign-checkpoint.dream3d"), checkpoint); + } + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6217); + CHECK(result.errors().front().message.find("foreign-checkpoint.dream3d") != std::string::npos); + CHECK((originalDestination.getAllDataObjectIds() == std::vector{1})); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK((originalDestination.getAllDataPaths() == std::vector{DataPath({"OriginalMarker"})})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"})})); + CHECK(originalDestination.getNextId() == 2); + CHECK(destination.getNextId() == 2); + CHECK(originalDestination.getData(DataObject::IdType{1}) == originalMarker.get()); + CHECK(destination.getData(DataObject::IdType{1}) == marker.get()); + CHECK(marker->getIDataStore() == markerStorePtr); + CHECK((*marker)[0] == 51); + CHECK((*marker)[1] == 52); + CHECK(prepared.getData(DataObject::IdType{2}) == candidate.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK(candidate->getIDataStore() == candidateStorePtr); + CHECK(prepared.getNextId() == 3); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(originalDestination); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: stale checkpoints preserve successful imports and ordinary insertions", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + const bool previousPublication = GENERATE(false, true); + const bool instrumented = GENERATE(false, true); + CAPTURE(previousPublication, instrumented); + DataStructure destination; + auto marker = createImportPublicationArray(destination, "Marker", 1, std::array{51, 52}); + const auto* markerStorePtr = marker->getIDataStore(); + const DataStructure::ImportPublicationCheckpoint originalCheckpoint(destination); + const auto checkpoint = originalCheckpoint; + const std::string retainedName = previousPublication ? "ImportedA" : "Unrelated"; + if(previousPublication) + { + DataStructure firstPrepared; + createImportPublicationArray(firstPrepared, retainedName, 2, std::array{61, 62}); + const auto first = destination.publishImportedObjects(firstPrepared, {DataPath({retainedName})}, fs::path("first-publication.dream3d"), originalCheckpoint); + SIMPLNX_RESULT_REQUIRE_VALID(first); + } + else + { + auto store = std::make_shared>(ShapeType{2}, ShapeType{1}, 0); + (*store)[0] = 61; + (*store)[1] = 62; + REQUIRE(Int32Array::Create(destination, retainedName, store) != nullptr); + } + auto retained = std::dynamic_pointer_cast(destination.getSharedData(DataObject::IdType{2})); + REQUIRE(retained != nullptr); + const auto* retainedStorePtr = retained->getIDataStore(); + REQUIRE(destination.getNextId() == 3); + DataStructure prepared; + auto candidate = createImportPublicationArray(prepared, "ImportedB", 3, std::array{31, -7}); + const auto* candidateStorePtr = candidate->getIDataStore(); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::BeforePublication, false); + usize notifications = 0; + const nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr&) { ++notifications; })); + Result<> result; + if(instrumented) + { + result = control.publish(destination, prepared, {DataPath({"ImportedB"})}, fs::path("stale-checkpoint.dream3d"), checkpoint); + } + else + { + result = destination.publishImportedObjects(prepared, {DataPath({"ImportedB"})}, fs::path("stale-checkpoint.dream3d"), checkpoint); + } + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -6218); + CHECK(result.errors().front().message.find("next identifier is 3") != std::string::npos); + CHECK(result.errors().front().message.find("checkpoint recorded 2") != std::string::npos); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2})); + CHECK((destination.getAllDataPaths() == std::vector{DataPath({"Marker"}), DataPath({retainedName})})); + CHECK(destination.getNextId() == 3); + CHECK(destination.getData(DataObject::IdType{1}) == marker.get()); + CHECK(destination.getData(DataObject::IdType{2}) == retained.get()); + CHECK(destination.getData(DataPath({retainedName})) == retained.get()); + CHECK_FALSE(destination.containsData(DataPath({"ImportedB"}))); + CHECK(marker->getIDataStore() == markerStorePtr); + CHECK(retained->getIDataStore() == retainedStorePtr); + CHECK((*marker)[0] == 51); + CHECK((*marker)[1] == 52); + CHECK((*retained)[0] == 61); + CHECK((*retained)[1] == 62); + CHECK(prepared.getData(DataObject::IdType{3}) == candidate.get()); + CHECK(candidate->getDataStructure() == &prepared); + CHECK(candidate->getIDataStore() == candidateStorePtr); + CHECK(prepared.getNextId() == 4); + CHECK(notifications == 0); + CHECK_FALSE(control.publicationFaultConsumed()); + CHECK_FALSE(control.cleanupFaultConsumed()); + CHECK(control.retainedOwner() == nullptr); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} + +TEST_CASE("Import publication: complete rollback permits checkpoint reuse without stale-owner deletion", "[ImportPublication][DataStructure]") +{ + UnitTest::LoadPlugins(); + DataStructure destination; + auto marker = createImportPublicationArray(destination, "Marker", 1, std::array{51, 52}); + const DataStructure::ImportPublicationCheckpoint checkpoint(destination); + DataStructure prepared; + auto firstSource = createImportPublicationArray(prepared, "First", 2, std::array{31, -7}); + auto secondSource = createImportPublicationArray(prepared, "Second", 3, std::array{81, 82}); + const std::vector selected{DataPath({"First"}), DataPath({"Second"})}; + std::vector removals; + usize additions = 0; + const nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr& message) { + if(message->getMsgType() == DataRemovedMessage::MsgType) + { + removals.push_back(std::static_pointer_cast(message)->getId()); + } + else if(message->getMsgType() == DataAddedMessage::MsgType) + { + ++additions; + } + })); + DataStructure::ImportPublicationTestControl control; + control.setFault(DataStructure::ImportPublicationTestControl::Fault::AfterSecondInsert, false); + const auto failed = control.publish(destination, prepared, selected, fs::path("complete-rollback.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_INVALID(failed); + REQUIRE(failed.errors().size() == 1); + CHECK(failed.errors().front().code == -6214); + REQUIRE(control.publicationFaultConsumed()); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK(destination.getData(DataObject::IdType{1}) == marker.get()); + CHECK(destination.getNextId() == 2); + CHECK((removals == std::vector{3, 2})); + auto detachedOwner = control.retainedOwner(); + REQUIRE(detachedOwner != nullptr); + CHECK(detachedOwner->getId() == 3); + CHECK(detachedOwner->getDataStructure() == nullptr); + CHECK(detachedOwner->getParentIds().empty()); + const auto retry = control.publish(destination, prepared, selected, fs::path("complete-rollback.dream3d"), checkpoint); + SIMPLNX_RESULT_REQUIRE_VALID(retry); + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2, 3})); + CHECK(destination.getNextId() == 4); + auto first = std::dynamic_pointer_cast(destination.getSharedData(DataObject::IdType{2})); + auto second = std::dynamic_pointer_cast(destination.getSharedData(DataObject::IdType{3})); + REQUIRE(first != nullptr); + REQUIRE(second != nullptr); + CHECK(first.get() != firstSource.get()); + CHECK(second.get() != secondSource.get()); + CHECK(second.get() != detachedOwner.get()); + CHECK(first->getIDataStore() == firstSource->getIDataStore()); + CHECK(second->getIDataStore() == secondSource->getIDataStore()); + CHECK((*first)[0] == 31); + CHECK((*first)[1] == -7); + CHECK((*second)[0] == 81); + CHECK((*second)[1] == 82); + detachedOwner.reset(); + CHECK(destination.getData(DataObject::IdType{3}) == second.get()); + CHECK((removals == std::vector{3, 2})); + CHECK(additions == 0); + UnitTest::CheckArraysInheritTupleDims(destination); + UnitTest::CheckArraysInheritTupleDims(prepared); +} +#endif diff --git a/test/Dream3dLoadingApiTest.cpp b/test/Dream3dLoadingApiTest.cpp index cb9f71a7cf..e8b47e2079 100644 --- a/test/Dream3dLoadingApiTest.cpp +++ b/test/Dream3dLoadingApiTest.cpp @@ -1667,6 +1667,7 @@ TEST_CASE("ImportH5ObjectPathsAction preserves recovery warnings through merging const auto application = Application::GetOrCreateInstance(); const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); const int mergeCase = GENERATE(0, 1, 2); + CAPTURE(mergeCase); const auto source = CreateNonnumericSelectionTestDataStructure(); const auto fileName = std::string("Dream3dLoadingApiTest_ActionWarning_") + std::to_string(mergeCase) + ".dream3d"; const auto filePath = WriteTestFile(source, fileName); @@ -1679,19 +1680,29 @@ TEST_CASE("ImportH5ObjectPathsAction preserves recovery warnings through merging { REQUIRE(DataGroup::Create(destination, k_NonnumericGroupName) != nullptr); } + const auto originalPaths = destination.getAllDataPaths(); + const auto originalIds = destination.getAllDataObjectIds(); + const auto originalNextId = destination.getNextId(); + const auto originalGroup = destination.getSharedData(k_NonnumericGroupPath); const std::vector paths = mergeCase == 2 ? std::vector{k_NonnumericNumericArrayPath} : std::vector{k_NonnumericGroupPath, k_NonnumericAttrMatPath, k_NonnumericNumericArrayPath, k_NonnumericRecoverySiblingPath}; const ImportH5ObjectPathsAction action(filePath, paths); const auto result = action.apply(destination, IDataAction::Mode::Execute); - REQUIRE(ContainsWarning(result, -89200)); if(mergeCase != 0) { REQUIRE(result.invalid()); REQUIRE(result.errors().front().code == (mergeCase == 1 ? -6203 : -6202)); + // Destination validation rejects these imports before the malformed array is read. + CHECK_FALSE(ContainsWarning(result, -89200)); + CHECK(destination.getAllDataPaths() == originalPaths); + CHECK(destination.getAllDataObjectIds() == originalIds); + CHECK(destination.getNextId() == originalNextId); + CHECK(destination.getSharedData(k_NonnumericGroupPath) == originalGroup); } else { REQUIRE(result.valid()); + REQUIRE(ContainsWarning(result, -89200)); REQUIRE_NOTHROW(destination.getDataRefAs(k_NonnumericRecoverySiblingPath)); const auto& sibling = destination.getDataRefAs(k_NonnumericRecoverySiblingPath); CHECK(sibling.getValue(0) == 3); @@ -1700,6 +1711,7 @@ TEST_CASE("ImportH5ObjectPathsAction preserves recovery warnings through merging REQUIRE_NOTHROW(destination.getDataRefAs(k_NonnumericNumericArrayPath)); CHECK(destination.getDataRefAs(k_NonnumericNumericArrayPath).getStoreType() == IDataStore::StoreType::Empty); } + UnitTest::CheckArraysInheritTupleDims(destination); } TEST_CASE("ImportH5ObjectPathsAction avoids excluded materialization", "[simplnx][Dream3dLoadingApi][SelectiveImport]") diff --git a/test/Dream3dPreflightCacheTest.cpp b/test/Dream3dPreflightCacheTest.cpp index d4e47e6ed2..23f04ba2a7 100644 --- a/test/Dream3dPreflightCacheTest.cpp +++ b/test/Dream3dPreflightCacheTest.cpp @@ -17,6 +17,7 @@ #include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" #include "simplnx/DataStructure/NeighborList.hpp" #include "simplnx/DataStructure/StringArray.hpp" +#include "simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp" #include "simplnx/Pipeline/Pipeline.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" @@ -56,6 +57,7 @@ class CacheRecordingResolver : public IDataStoreFormatResolver public: std::string selectedFormat; std::string failureMessage; + std::optional failurePath; mutable std::vector paths; mutable std::vector types; mutable std::vector bytes; @@ -65,7 +67,7 @@ class CacheRecordingResolver : public IDataStoreFormatResolver paths.push_back(path); types.push_back(numericType); bytes.push_back(logicalBytes); - if(!failureMessage.empty()) + if(!failureMessage.empty() && (!failurePath.has_value() || path == *failurePath)) { throw std::runtime_error(failureMessage); } @@ -802,3 +804,329 @@ TEST_CASE("StorageFormatPlan: disk warnings survive cache bypass preparation", " CHECK(hasLegacyWarning(failedStat)); #endif } + +TEST_CASE("C8 neutral metadata bypasses process policy on every cache route", "[C8][Dream3dPreflightCache]") +{ + CacheTestContext context; + const int route = GENERATE(0, 1, 2); + CAPTURE(route); + const auto path = WriteTestFile("neutral_cache_routes.dream3d", 4); + const auto cleanup = MakeScopeGuard([&path]() noexcept { + std::error_code error; + fs::remove(path, error); + }); + if(route == 1) + { + fs::last_write_time(path, fs::file_time_type::clock::now()); + } +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + DREAM3D::Dream3dPreflightCache::SetForceFileMetadataFailure(route == 2); +#endif + context.resolver->clearRecords(); + context.resolver->failureMessage = "neutral cache must bypass process policy"; + auto first = context.cache.fetchNeutralMetadata(path); + auto second = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(first); + SIMPLNX_RESULT_REQUIRE_VALID(second); + CHECK(first.value().fileVersion == DREAM3D::k_CurrentFileVersion); + CHECK(second.value().fileVersion == first.value().fileVersion); + CHECK(context.resolver->paths.empty()); + CHECK(context.cache.missCount() == (route == 0 ? 1 : 2)); + CHECK(context.cache.hitCount() == (route == 0 ? 1 : 0)); + const DataPath arrayPath({"TestGroup", "CellData", "Ints"}); + const auto& firstArray = RequirePlannedArray(first.value().dataStructure, arrayPath, "", {4}, {1}); + const auto& secondArray = RequirePlannedArray(second.value().dataStructure, arrayPath, "", {4}, {1}); + CHECK(firstArray.getIDataStore() != secondArray.getIDataStore()); + CHECK(firstArray.getId() == secondArray.getId()); + CHECK(first.value().dataStructure.getData(firstArray.getId()) == &firstArray); + CHECK(second.value().dataStructure.getData(secondArray.getId()) == &secondArray); + UnitTest::CheckArraysInheritTupleDims(second.value().dataStructure); +} + +TEST_CASE("C8 neutral and ordinary cache modes stay separate and invalidate together", "[C8][Dream3dPreflightCache]") +{ + CacheTestContext context; + const auto path = WriteTestFile("neutral_cache_mode_separation.dream3d", 4); + const auto cleanup = MakeScopeGuard([&path]() noexcept { + std::error_code error; + fs::remove(path, error); + }); + REQUIRE(context.cache.fetch(path).valid()); + context.resolver->clearRecords(); + context.resolver->failureMessage = "ordinary mode still resolves policy"; + REQUIRE(context.cache.fetchNeutralMetadata(path).valid()); + REQUIRE(context.cache.fetchNeutralMetadata(path).valid()); + CHECK(context.resolver->paths.empty()); + CHECK(context.cache.missCount() == 2); + CHECK(context.cache.hitCount() == 1); + auto ordinary = context.cache.fetch(path); + REQUIRE(ordinary.invalid()); + CHECK_FALSE(context.resolver->paths.empty()); + CHECK(context.cache.hitCount() == 2); + context.resolver->failureMessage.clear(); + context.cache.invalidate(path); + REQUIRE(context.cache.fetch(path).valid()); + REQUIRE(context.cache.fetchNeutralMetadata(path).valid()); + CHECK(context.cache.missCount() == 4); +} + +TEST_CASE("C8 neutral legacy metadata replays warnings and isolates eager values", "[C8][Dream3dPreflightCache]") +{ + CacheTestContext context; + const int route = GENERATE(0, 1, 2); + CAPTURE(route); + const auto path = WriteLegacyStatisticsWarningFile("neutral_cache_legacy_warnings.dream3d"); + const auto cleanup = MakeScopeGuard([&path]() noexcept { + std::error_code error; + fs::remove(path, error); + }); + context.resolver->failureMessage = "legacy neutral metadata must bypass process policy"; + context.resolver->clearRecords(); + if(route == 1) + { + fs::last_write_time(path, fs::file_time_type::clock::now()); + } +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS + DREAM3D::Dream3dPreflightCache::SetForceFileMetadataFailure(route == 2); +#endif + const DataPath valuesPath({"StatsContainer", "Statistics", "Values_Mean"}); + auto cold = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(cold); + REQUIRE(cold.value().fileVersion == DREAM3D::k_LegacyFileVersion); + REQUIRE(cold.warnings().size() == 1); + CHECK(cold.warnings().front().code == -298012); + auto* first = cold.value().dataStructure.getDataAs(valuesPath); + REQUIRE(first != nullptr); + REQUIRE(first->getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(first->size() == 3); + CHECK((*first)[0] == 4.0F); + CHECK((*first)[1] == 5.0F); + CHECK((*first)[2] == 6.0F); + (*first)[0] = 99.0F; + auto warm = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(warm); + REQUIRE(warm.value().fileVersion == DREAM3D::k_LegacyFileVersion); + REQUIRE(warm.warnings().size() == cold.warnings().size()); + for(usize index = 0; index < cold.warnings().size(); ++index) + { + CHECK(warm.warnings()[index].code == cold.warnings()[index].code); + CHECK(warm.warnings()[index].message == cold.warnings()[index].message); + } + const auto* second = warm.value().dataStructure.getDataAs(valuesPath); + REQUIRE(second != nullptr); + REQUIRE(second->size() == 3); + REQUIRE(first->getIDataStore() != second->getIDataStore()); + CHECK((*second)[0] == 4.0F); + CHECK((*second)[1] == 5.0F); + CHECK((*second)[2] == 6.0F); + CHECK(context.resolver->paths.empty()); + CHECK(context.cache.hitCount() == (route == 0 ? 1 : 0)); + CHECK(context.cache.missCount() == (route == 0 ? 1 : 2)); + UnitTest::CheckArraysInheritTupleDims(warm.value().dataStructure); +} + +TEST_CASE("C8 neutral cache isolates list and string placeholders", "[C8][Dream3dPreflightCache]") +{ + CacheTestContext context; + DataStructure source; + source.setFormatResolver(context.inMemoryResolver); + auto* lists = NeighborList::Create(source, "Lists", ShapeType{3}); + REQUIRE(lists != nullptr); + lists->setList(0, std::vector{3, 6}); + REQUIRE(StringArray::CreateWithValues(source, "Strings", ShapeType{2}, {"alpha", "beta"}) != nullptr); + const fs::path path = fs::path(unit_test::k_BinaryTestOutputDir.view()) / "neutral_list_string.dream3d"; + const auto cleanup = MakeScopeGuard([&path]() noexcept { + std::error_code error; + fs::remove(path, error); + }); + auto write = DREAM3D::WriteFile(path, source); + SIMPLNX_RESULT_REQUIRE_VALID(write); + AgeFile(path); + context.resolver->clearRecords(); + context.resolver->failureMessage = "neutral list metadata must bypass process policy"; + auto first = context.cache.fetchNeutralMetadata(path); + auto second = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(first); + SIMPLNX_RESULT_REQUIRE_VALID(second); + auto* firstList = first.value().dataStructure.getDataAs>(DataPath({"Lists"})); + const auto* secondList = second.value().dataStructure.getDataAs>(DataPath({"Lists"})); + auto* firstString = first.value().dataStructure.getDataAs(DataPath({"Strings"})); + const auto* secondString = second.value().dataStructure.getDataAs(DataPath({"Strings"})); + REQUIRE(firstList != nullptr); + REQUIRE(secondList != nullptr); + REQUIRE(firstString != nullptr); + REQUIRE(secondString != nullptr); + CHECK(firstList->getStore() != secondList->getStore()); + REQUIRE(dynamic_cast*>(secondList->getStore().get()) != nullptr); + REQUIRE(firstString->isPlaceholder()); + REQUIRE(secondString->isPlaceholder()); + REQUIRE(firstList->getStore()->resizeTuples({5}).valid()); + REQUIRE(firstString->resizeTuples({5}).valid()); + CHECK(secondList->getTupleShape() == ShapeType{3}); + CHECK(secondString->getTupleShape() == ShapeType{2}); + CHECK(context.resolver->paths.empty()); + UnitTest::CheckArraysInheritTupleDims(second.value().dataStructure); +} + +TEST_CASE("C8 NX Statistics remains deferred metadata", "[C8][Dream3dPreflightCache]") +{ + CacheTestContext context; + DataStructure source; + source.setFormatResolver(context.inMemoryResolver); + auto* container = DataGroup::Create(source, "StatsContainer"); + REQUIRE(container != nullptr); + auto* statistics = DataGroup::Create(source, "Statistics", container->getId()); + REQUIRE(statistics != nullptr); + auto values = std::make_shared>(ShapeType{3}, ShapeType{1}, 17.0F); + REQUIRE(Float32Array::Create(source, "Values_Mean", values, statistics->getId()) != nullptr); + const fs::path path = fs::path(unit_test::k_BinaryTestOutputDir.view()) / "c8_nx_statistics.dream3d"; + const auto cleanup = MakeScopeGuard([&path]() noexcept { + std::error_code error; + fs::remove(path, error); + }); + auto write = DREAM3D::WriteFile(path, source); + SIMPLNX_RESULT_REQUIRE_VALID(write); + AgeFile(path); + context.resolver->failureMessage = "NX Statistics metadata must not consult process policy"; + context.resolver->clearRecords(); + const DataPath valuesPath({"StatsContainer", "Statistics", "Values_Mean"}); + auto cold = context.cache.fetchNeutralMetadata(path); + auto warm = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(cold); + SIMPLNX_RESULT_REQUIRE_VALID(warm); + REQUIRE(cold.value().fileVersion == DREAM3D::k_CurrentFileVersion); + REQUIRE(warm.value().fileVersion == DREAM3D::k_CurrentFileVersion); + const auto& first = RequirePlannedArray(cold.value().dataStructure, valuesPath, "", {3}, {1}); + const auto& second = RequirePlannedArray(warm.value().dataStructure, valuesPath, "", {3}, {1}); + CHECK(first.getIDataStore() != second.getIDataStore()); + CHECK(cold.warnings().empty()); + CHECK(warm.warnings().empty()); + CHECK(context.resolver->paths.empty()); + + DataStructure destination; + destination.setFormatResolver(context.inMemoryResolver); + ImportH5ObjectPathsAction action(path, {DataPath({"StatsContainer"}), DataPath({"StatsContainer", "Statistics"}), valuesPath}); + auto preflight = action.apply(destination, IDataAction::Mode::Preflight); + SIMPLNX_RESULT_REQUIRE_VALID(preflight); + RequirePlannedArray(destination, valuesPath, "", {3}, {1}); + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("C8 legacy Statistics action preserves literal values and warning on planning failure", "[C8][Dream3dPreflightCache][StorageFormatPlan]") +{ + CacheTestContext context; + const bool warmCache = GENERATE(false, true); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool failDestination = GENERATE(false, true); + CAPTURE(warmCache, mode, failDestination); + const auto path = WriteLegacyStatisticsWarningFile("c8_legacy_statistics_action.dream3d"); + const auto cleanup = MakeScopeGuard([&path]() noexcept { + std::error_code error; + fs::remove(path, error); + }); + if(warmCache) + { + auto warm = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(warm); + } + context.resolver->failureMessage = "legacy action must not consult process policy"; + context.resolver->clearRecords(); + const DataPath valuesPath({"StatsContainer", "Statistics", "Values_Mean"}); + const DataPath datasetPath({"StatsContainer", "Statistics", "Values"}); + auto destinationResolver = std::make_shared(); + if(failDestination) + { + destinationResolver->failureMessage = "injected legacy destination planning failure"; + destinationResolver->failurePath = datasetPath; + } + DataStructure destination; + destination.setFormatResolver(destinationResolver); + auto* marker = DataGroup::Create(destination, "DestinationMarker"); + REQUIRE(marker != nullptr); + const auto markerId = marker->getId(); + const auto originalPaths = destination.getAllDataPaths(); + ImportH5ObjectPathsAction action(path, {DataPath({"StatsContainer"}), DataPath({"StatsContainer", "Statistics"}), valuesPath, datasetPath}); + auto result = action.apply(destination, mode); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings().front().code == -298012); + CHECK(context.resolver->paths.empty()); + const auto attributeDecisions = std::count(destinationResolver->paths.begin(), destinationResolver->paths.end(), valuesPath); + const auto datasetDecisions = std::count(destinationResolver->paths.begin(), destinationResolver->paths.end(), datasetPath); + CHECK(datasetDecisions == 1); + CHECK(destinationResolver->paths.size() == static_cast(attributeDecisions + datasetDecisions)); + CHECK(destination.getData(markerId) == marker); + if(failDestination) + { + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().message.find(destinationResolver->failureMessage) != std::string::npos); + CHECK(result.errors().front().message.find(datasetPath.toString()) != std::string::npos); + CHECK(attributeDecisions <= 1); + CHECK(destination.getAllDataPaths() == originalPaths); + } + else + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(attributeDecisions == 1); + auto* values = destination.getDataAs(valuesPath); + auto* dataset = destination.getDataAs(datasetPath); + REQUIRE(values != nullptr); + REQUIRE(dataset != nullptr); + REQUIRE(values->getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(dataset->getStoreType() == IDataStore::StoreType::InMemory); + REQUIRE(values->size() == 3); + REQUIRE(dataset->size() == 1); + CHECK((*dataset)[0] == 1.0F); + CHECK((*values)[0] == 4.0F); + CHECK((*values)[1] == 5.0F); + CHECK((*values)[2] == 6.0F); + (*values)[0] = -99.0F; + (*dataset)[0] = -88.0F; + auto laterHandout = context.cache.fetchNeutralMetadata(path); + SIMPLNX_RESULT_REQUIRE_VALID(laterHandout); + const auto* original = laterHandout.value().dataStructure.getDataAs(valuesPath); + const auto* originalDataset = laterHandout.value().dataStructure.getDataAs(datasetPath); + REQUIRE(original != nullptr); + REQUIRE(originalDataset != nullptr); + REQUIRE(original->getIDataStore() != values->getIDataStore()); + REQUIRE(originalDataset->getIDataStore() != dataset->getIDataStore()); + REQUIRE(originalDataset->size() == 1); + CHECK((*originalDataset)[0] == 1.0F); + CHECK((*original)[0] == 4.0F); + CHECK((*original)[1] == 5.0F); + CHECK((*original)[2] == 6.0F); + UnitTest::CheckArraysInheritTupleDims(destination); + } +} + +TEST_CASE("C8 neutral and ordinary cache masters share one LRU capacity", "[C8][Dream3dPreflightCache]") +{ + CacheTestContext context; + std::vector files; + const auto cleanup = MakeScopeGuard([&files]() noexcept { + for(const auto& path : files) + { + std::error_code error; + fs::remove(path, error); + } + }); + for(usize fileIdx = 0; fileIdx < DREAM3D::Dream3dPreflightCache::k_Capacity; ++fileIdx) + { + files.push_back(WriteTestFile(fmt::format("c8_mixed_cache_capacity_{}.dream3d", fileIdx), 4)); + } + REQUIRE(context.cache.fetchNeutralMetadata(files.front()).valid()); + REQUIRE(context.cache.fetch(files.front()).valid()); + for(usize fileIdx = 1; fileIdx + 1 < files.size(); ++fileIdx) + { + REQUIRE(context.cache.fetch(files[fileIdx]).valid()); + } + REQUIRE(context.cache.missCount() == DREAM3D::Dream3dPreflightCache::k_Capacity); + REQUIRE(context.cache.fetchNeutralMetadata(files.back()).valid()); + context.cache.resetStats(); + REQUIRE(context.cache.fetch(files.front()).valid()); + CHECK(context.cache.hitCount() == 1); + CHECK(context.cache.missCount() == 0); + REQUIRE(context.cache.fetchNeutralMetadata(files.front()).valid()); + CHECK(context.cache.hitCount() == 1); + CHECK(context.cache.missCount() == 1); +} diff --git a/test/FloatArrayComparisonTest.cpp b/test/FloatArrayComparisonTest.cpp new file mode 100644 index 0000000000..047ffc9a8d --- /dev/null +++ b/test/FloatArrayComparisonTest.cpp @@ -0,0 +1,381 @@ +#include "simplnx/Common/Result.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +constexpr int32 k_InjectedReadError = -98781; +constexpr int32 k_InjectedReadWarning = 98782; +const DataPath k_ExemplarPath({"Exemplar"}); +const DataPath k_ComputedPath({"Computed"}); + +/** + * @class RecordingFloatStore + * @brief Records real resident reads and can fail one selected call. + * @tparam T Floating-point scalar type. + */ +template +class RecordingFloatStore : public DataStore +{ +public: + using DataStore::DataStore; + + ~RecordingFloatStore() override = default; + RecordingFloatStore(const RecordingFloatStore&) = default; + RecordingFloatStore(RecordingFloatStore&&) = default; + RecordingFloatStore& operator=(const RecordingFloatStore&) = delete; + RecordingFloatStore& operator=(RecordingFloatStore&&) = default; + + mutable std::vector> reads; + usize failingReadCall = 0; + bool warnOnRead = false; + + /** + * @brief Records each range before reading values or returning the selected error. + * @param startIndex First flat value index. + * @param buffer Receives real resident values on success. + * @return Known injected error or the resident read result, with an optional known warning. + */ + Result<> copyIntoBuffer(usize startIndex, nonstd::span buffer) const override + { + reads.emplace_back(startIndex, buffer.size()); + auto result = reads.size() == failingReadCall ? MakeErrorResult(k_InjectedReadError, "Injected floating-array read error") : DataStore::copyIntoBuffer(startIndex, buffer); + if(warnOnRead) + { + result.warnings().push_back({k_InjectedReadWarning, "Injected floating-array read warning"}); + } + return result; + } +}; + +/** + * @brief Checks every stored value without calling the comparison helper or its bulk read path. + * @tparam T Floating-point scalar type. + * @param store Resident values to verify. + * @param expected Independent snapshot supplied by the fixture. + */ +template +void CheckPreservedValues(const RecordingFloatStore& store, const std::vector& expected) +{ + REQUIRE(store.getSize() == expected.size()); + for(usize valueIndex = 0; valueIndex < expected.size(); ++valueIndex) + { + CAPTURE(valueIndex); + const T actualValue = store.getValue(valueIndex); + if(std::isnan(expected[valueIndex])) + { + CHECK(std::isnan(actualValue)); + } + else + { + CHECK(actualValue == expected[valueIndex]); + CHECK(std::signbit(actualValue) == std::signbit(expected[valueIndex])); + } + } +} + +/** + * @struct FloatComparisonFixture + * @brief Owns two real arrays and independent copies of all their original values. + * @tparam T Floating-point scalar type. + */ +template +struct FloatComparisonFixture +{ + DataStructure dataStructure; + std::vector exemplaryValues; + std::vector computedValues; + std::shared_ptr> exemplaryStore; + std::shared_ptr> computedStore; + DataArray* exemplaryArray = nullptr; + DataArray* computedArray = nullptr; + + /** + * @brief Creates and initializes both resident arrays before a test can enable read failures. + * @param exemplary Exemplar snapshot. + * @param computed Computed snapshot. + * @param exemplaryTuples Exemplar tuple shape; an empty shape selects one tuple per value. + * @param exemplaryComponents Exemplar component shape. + * @param computedTuples Computed tuple shape; an empty shape selects one tuple per value. + * @param computedComponents Computed component shape. + */ + FloatComparisonFixture(std::vector exemplary, std::vector computed, ShapeType exemplaryTuples = {}, ShapeType exemplaryComponents = {1}, ShapeType computedTuples = {}, + ShapeType computedComponents = {1}) + : exemplaryValues(std::move(exemplary)) + , computedValues(std::move(computed)) + { + if(exemplaryTuples.empty()) + { + exemplaryTuples = {exemplaryValues.size()}; + } + if(computedTuples.empty()) + { + computedTuples = {computedValues.size()}; + } + exemplaryStore = std::make_shared>(exemplaryTuples, exemplaryComponents, T{}); + computedStore = std::make_shared>(computedTuples, computedComponents, T{}); + REQUIRE(exemplaryStore->getSize() == exemplaryValues.size()); + REQUIRE(computedStore->getSize() == computedValues.size()); + exemplaryArray = DataArray::Create(dataStructure, k_ExemplarPath.getTargetName(), exemplaryStore); + computedArray = DataArray::Create(dataStructure, k_ComputedPath.getTargetName(), computedStore); + REQUIRE(exemplaryArray != nullptr); + REQUIRE(computedArray != nullptr); + auto exemplaryWriteResult = exemplaryStore->copyFromBuffer(0, nonstd::span(exemplaryValues.data(), exemplaryValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(exemplaryWriteResult); + auto computedWriteResult = computedStore->copyFromBuffer(0, nonstd::span(computedValues.data(), computedValues.size())); + SIMPLNX_RESULT_REQUIRE_VALID(computedWriteResult); + requireUnchanged(); + REQUIRE(exemplaryStore->reads.empty()); + REQUIRE(computedStore->reads.empty()); + } + + ~FloatComparisonFixture() = default; + FloatComparisonFixture(const FloatComparisonFixture&) = default; + FloatComparisonFixture(FloatComparisonFixture&&) = default; + FloatComparisonFixture& operator=(const FloatComparisonFixture&) = default; + FloatComparisonFixture& operator=(FloatComparisonFixture&&) = default; + + /** + * @brief Evaluates the actual arrays and verifies that every source value stays unchanged. + * @param epsilon Exclusive absolute-difference limit. + * @param checkNans Selects the NaN-position policy. + * @return Actual evaluator result, including any setup or read error. + */ + Result> evaluate(T epsilon, bool checkNans) + { + auto result = UnitTest::FloatArrayComparisonDetail::EvaluateFloatArraysWithNans(*exemplaryArray, *computedArray, epsilon, checkNans); + requireUnchanged(); + return result; + } + + /** + * @brief Verifies full source values and the fixture's array hierarchy. + */ + void requireUnchanged() const + { + CheckPreservedValues(*exemplaryStore, exemplaryValues); + CheckPreservedValues(*computedStore, computedValues); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +}; +} // namespace + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: equal values and public wrapper", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + const TestType nanValue = std::numeric_limits::quiet_NaN(); + FloatComparisonFixture fixture({0, TestType{-0.0}, 2, nanValue}, {0, TestType{-0.0}, 2, nanValue}); + auto result = fixture.evaluate(TestType{0.5}, true); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.value().matches); + CHECK(result.value().maximumDifference == TestType{0}); + CHECK(result.value().maximumDifferenceIndex == 0); + CHECK_FALSE(result.value().firstNanMismatchIndex.has_value()); + UnitTest::CompareFloatArraysWithNans(fixture.dataStructure, k_ExemplarPath, k_ComputedPath); + fixture.requireUnchanged(); +} + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: NaN positions and full scan", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + const bool reverseInputs = GENERATE(false, true); + const bool differentNanPositions = GENERATE(false, true); + const bool checkNans = GENERATE(false, true); + const bool finiteMismatch = GENERATE(false, true); + const TestType nanValue = std::numeric_limits::quiet_NaN(); + std::vector exemplary = {1, nanValue, 3, 4, 5}; + std::vector computed = {1, 2, differentNanPositions ? nanValue : TestType{3}, finiteMismatch ? TestType{6} : TestType{4}, finiteMismatch ? TestType{8} : TestType{5}}; + if(reverseInputs) + { + std::swap(exemplary, computed); + } + CAPTURE(reverseInputs, differentNanPositions, checkNans, finiteMismatch); + FloatComparisonFixture fixture(exemplary, computed); + auto result = fixture.evaluate(TestType{0.5}, checkNans); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.value().matches == (!checkNans && !finiteMismatch)); + if(checkNans) + { + REQUIRE(result.value().firstNanMismatchIndex == std::optional{1}); + CHECK(std::isnan(result.value().exemplaryNanMismatchValue) == !reverseInputs); + CHECK(std::isnan(result.value().generatedNanMismatchValue) == reverseInputs); + CHECK((reverseInputs ? result.value().exemplaryNanMismatchValue : result.value().generatedNanMismatchValue) == TestType{2}); + } + else + { + CHECK_FALSE(result.value().firstNanMismatchIndex.has_value()); + } + CHECK(result.value().maximumDifference == (finiteMismatch ? TestType{3} : TestType{0})); + CHECK(result.value().maximumDifferenceIndex == (finiteMismatch ? 4 : 0)); + if(!checkNans && !finiteMismatch) + { + UnitTest::CompareFloatArraysWithNans(fixture.dataStructure, k_ExemplarPath, k_ComputedPath, TestType{0.5}, false); + fixture.requireUnchanged(); + } +} + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: exclusive tolerance and maximum", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + const TestType difference = GENERATE(TestType{0.25}, TestType{0.5}, TestType{0.75}); + FloatComparisonFixture fixture({0, 0, 0, 0}, {TestType{0.125}, difference, TestType{0.125}, difference}); + auto result = fixture.evaluate(TestType{0.5}, true); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.value().matches == (difference < TestType{0.5})); + CHECK(result.value().maximumDifference == difference); + CHECK(result.value().maximumDifferenceIndex == 1); + CHECK_FALSE(result.value().firstNanMismatchIndex.has_value()); +} + +TEST_CASE("UnitTest::CompareFloatArraysWithNans: double precision", "[UnitTest][FloatArrayComparison]") +{ + const float64 difference = 1.0 + std::ldexp(1.0, -40); + const float64 epsilon = 1.0 + std::ldexp(1.0, -41); + FloatComparisonFixture fixture({0.0}, {difference}); + auto result = fixture.evaluate(epsilon, true); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK_FALSE(result.value().matches); + CHECK(result.value().maximumDifference == difference); + CHECK(result.value().maximumDifferenceIndex == 0); +} + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: infinity pairs", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + const int pairIndex = GENERATE(0, 1, 2, 3, 4, 5); + const TestType infinity = std::numeric_limits::infinity(); + const std::vector> pairs = {{infinity, infinity}, {-infinity, -infinity}, {infinity, -infinity}, + {-infinity, infinity}, {TestType{2}, infinity}, {infinity, TestType{2}}}; + CAPTURE(pairIndex); + FloatComparisonFixture fixture({pairs[pairIndex].first}, {pairs[pairIndex].second}); + auto result = fixture.evaluate(TestType{0.5}, true); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.value().matches == (pairIndex < 2)); + CHECK(result.value().maximumDifference == (pairIndex < 2 ? TestType{0} : infinity)); + CHECK_FALSE(result.value().firstNanMismatchIndex.has_value()); + if(pairIndex < 2) + { + UnitTest::CompareFloatArraysWithNans(fixture.dataStructure, k_ExemplarPath, k_ComputedPath); + fixture.requireUnchanged(); + } +} + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: bounded tail and read failures", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + const bool reverseInputs = GENERATE(false, true); + // The fault variants reuse this one large fixture to check a short final read on either input. + const int failingSide = GENERATE(0, 1, 2); + std::vector exemplary(40001, TestType{7}); + std::vector computed = exemplary; + exemplary[40000] = std::numeric_limits::quiet_NaN(); + if(reverseInputs) + { + std::swap(exemplary, computed); + } + CAPTURE(reverseInputs, failingSide); + FloatComparisonFixture fixture(exemplary, computed); + fixture.exemplaryStore->failingReadCall = failingSide == 1 ? 2 : 0; + fixture.computedStore->failingReadCall = failingSide == 2 ? 2 : 0; + fixture.exemplaryStore->warnOnRead = true; + fixture.computedStore->warnOnRead = true; + auto result = fixture.evaluate(TestType{0.5}, true); + const std::vector> fullReads = {{0, 40000}, {40000, 1}}; + const std::vector> firstRead = {{0, 40000}}; + CHECK(fixture.exemplaryStore->reads == fullReads); + CHECK(fixture.computedStore->reads == (failingSide == 1 ? firstRead : fullReads)); + REQUIRE(result.warnings().size() == (failingSide == 1 ? 3 : 4)); + for(const auto& warning : result.warnings()) + { + CHECK(warning.code == k_InjectedReadWarning); + CHECK(warning.message == "Injected floating-array read warning"); + } + if(failingSide == 0) + { + REQUIRE(result.valid()); + CHECK_FALSE(result.value().matches); + CHECK(result.value().firstNanMismatchIndex == std::optional{40000}); + CHECK(std::isnan(result.value().exemplaryNanMismatchValue) == !reverseInputs); + CHECK(std::isnan(result.value().generatedNanMismatchValue) == reverseInputs); + CHECK(result.value().maximumDifference == TestType{0}); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == k_InjectedReadError); + CHECK(result.errors()[0].message == "Injected floating-array read error"); + } +} + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: shape counts before reads", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + SECTION("Unequal tuple counts in either direction") + { + const bool reverseInputs = GENERATE(false, true); + const usize exemplaryCount = reverseInputs ? 3 : 2; + const usize computedCount = reverseInputs ? 2 : 3; + FloatComparisonFixture fixture(std::vector(exemplaryCount, 1), std::vector(computedCount, 1)); + auto result = fixture.evaluate(TestType{0.5}, true); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -90051); + CHECK(result.errors()[0].message == fmt::format("Floating-array comparison requires equal tuple counts: exemplar {}, computed {}.", exemplaryCount, computedCount)); + CHECK(fixture.exemplaryStore->reads.empty()); + CHECK(fixture.computedStore->reads.empty()); + } + SECTION("Fewer or more computed components") + { + const usize computedComponents = GENERATE(usize{1}, usize{3}); + FloatComparisonFixture fixture(std::vector(4, 1), std::vector(2 * computedComponents, 1), {2}, {2}, {2}, {computedComponents}); + auto result = fixture.evaluate(TestType{0.5}, true); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -90052); + CHECK(result.errors()[0].message == fmt::format("Floating-array comparison requires equal component counts: exemplar 2, computed {}.", computedComponents)); + CHECK(fixture.exemplaryStore->reads.empty()); + CHECK(fixture.computedStore->reads.empty()); + } + SECTION("Different shape vectors with equal tuple and component counts") + { + const std::vector values = {1, 2, 3, 4, 5, 6, 7, 8}; + FloatComparisonFixture fixture(values, values, {2, 2}, {2}, {4}, {1, 2}); + auto result = fixture.evaluate(TestType{0.5}, true); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.value().matches); + const std::vector> reads = {{0, 8}}; + CHECK(fixture.exemplaryStore->reads == reads); + CHECK(fixture.computedStore->reads == reads); + UnitTest::CompareFloatArraysWithNans(fixture.dataStructure, k_ExemplarPath, k_ComputedPath); + fixture.requireUnchanged(); + } +} + +TEMPLATE_TEST_CASE("UnitTest::CompareFloatArraysWithNans: empty arrays and unusual tolerances", "[UnitTest][FloatArrayComparison]", float32, float64) +{ + const int epsilonIndex = GENERATE(0, 1, 2, 3, 4); + const bool emptyArrays = GENERATE(false, true); + const std::vector epsilons = {TestType{0.5}, TestType{0}, TestType{-0.5}, std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity()}; + const std::vector values = emptyArrays ? std::vector{} : std::vector{1}; + CAPTURE(epsilonIndex, emptyArrays); + FloatComparisonFixture fixture(values, values); + auto result = fixture.evaluate(epsilons[epsilonIndex], true); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(result.value().matches == (epsilonIndex == 0 || epsilonIndex == 4)); + CHECK(result.value().maximumDifference == TestType{0}); + CHECK(result.value().maximumDifferenceIndex == 0); + if(emptyArrays) + { + CHECK(fixture.exemplaryStore->reads.empty()); + CHECK(fixture.computedStore->reads.empty()); + } +} diff --git a/test/GeometryBulkContractsTest.cpp b/test/GeometryBulkContractsTest.cpp index 1f9448ea75..86679b23b9 100644 --- a/test/GeometryBulkContractsTest.cpp +++ b/test/GeometryBulkContractsTest.cpp @@ -9,11 +9,15 @@ #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" #include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" +#include "simplnx/Utilities/DataStoreUtilities.hpp" +#include "simplnx/Utilities/GeometryUtilities.hpp" #include +#include #include #include +#include #include #include #include @@ -535,3 +539,85 @@ TEST_CASE("RectGrid element sizes return bulk failures and clear cached IDs", "[ CHECK_FALSE(geometryPtr->getElementSizesId().has_value()); UnitTest::CheckArraysInheritTupleDims(dataStructure); } + +TEST_CASE("GeometryUtilities computes triangle normals with float arithmetic", "[simplnx][GeometryBulkContracts][GeometryUtilities]") +{ + UnitTest::LoadPlugins(); + + const auto scenario = GENERATE(from_range(UnitTest::SelectAlgorithmTestScenariosForInMemoryStores())); + const bool preCancelled = GENERATE(false, true); + CAPTURE(scenario, preCancelled); + UnitTest::AlgorithmTestScope algorithmTestScope(scenario); + + const DataPath geometryPath({"NormalMesh"}); + const DataPath verticesPath = geometryPath.createChildPath("Vertices"); + const DataPath facesPath = geometryPath.createChildPath("Faces"); + const DataPath normalsPath = geometryPath.createChildPath("Normals"); + DataStructure dataStructure; + auto* triangleGeomPtr = TriangleGeom::Create(dataStructure, geometryPath.getTargetName()); + REQUIRE(triangleGeomPtr != nullptr); + + auto verticesStore = DataStoreUtilities::CreateDataStore(dataStructure, verticesPath, {6}, {3}); + auto facesStore = DataStoreUtilities::CreateDataStore(dataStructure, facesPath, {4}, {3}); + auto normalsStore = DataStoreUtilities::CreateDataStore(dataStructure, normalsPath, {4}, {3}); + REQUIRE(verticesStore != nullptr); + REQUIRE(facesStore != nullptr); + REQUIRE(normalsStore != nullptr); + auto* verticesPtr = Float32Array::Create(dataStructure, verticesPath.getTargetName(), verticesStore, triangleGeomPtr->getId()); + auto* facesPtr = DataArray::Create(dataStructure, facesPath.getTargetName(), facesStore, triangleGeomPtr->getId()); + auto* normalsPtr = Float64Array::Create(dataStructure, normalsPath.getTargetName(), normalsStore, triangleGeomPtr->getId()); + REQUIRE(verticesPtr != nullptr); + REQUIRE(facesPtr != nullptr); + REQUIRE(normalsPtr != nullptr); + triangleGeomPtr->setVertices(*verticesPtr); + triangleGeomPtr->setFaceList(*facesPtr); + + algorithmTestScope.requireExpectedStore(*verticesPtr); + algorithmTestScope.requireExpectedStore(*facesPtr); + algorithmTestScope.requireExpectedStore(*normalsPtr); + + const std::array originalVertices = {0.0F, 0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 0.0F, 1.0F, 0.0F, 0.0F, 0.0F, 1.0F, 3.0F, 0.0F, 1.0F, 0.0F, 2.0F, 1.0F}; + const std::array originalFaces = {0, 1, 2, 0, 2, 1, 0, 2, 3, 0, 4, 5}; + auto verticesWriteResult = verticesStore->copyFromBuffer(0, nonstd::span(originalVertices.data(), originalVertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesWriteResult); + auto facesWriteResult = facesStore->copyFromBuffer(0, nonstd::span(originalFaces.data(), originalFaces.size())); + SIMPLNX_RESULT_REQUIRE_VALID(facesWriteResult); + constexpr float64 k_OutputSentinel = -37.0; + normalsStore->fill(k_OutputSentinel); + + const std::atomic_bool shouldCancel{preCancelled}; + auto result = algorithmTestScope.execute([&] { return GeometryUtilities::ComputeTriangleNormals(triangleGeomPtr, *normalsStore, shouldCancel); }); + SIMPLNX_RESULT_REQUIRE_VALID(result); + + REQUIRE(triangleGeomPtr->getVertices() == verticesPtr); + REQUIRE(triangleGeomPtr->getFaces() == facesPtr); + REQUIRE(verticesPtr->getTupleShape() == ShapeType{6}); + REQUIRE(verticesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(facesPtr->getTupleShape() == ShapeType{4}); + REQUIRE(facesPtr->getComponentShape() == ShapeType{3}); + REQUIRE(normalsPtr->getTupleShape() == ShapeType{4}); + REQUIRE(normalsPtr->getComponentShape() == ShapeType{3}); + + std::array actualVertices = {}; + std::array actualFaces = {}; + std::array actualNormals = {}; + auto verticesReadResult = verticesStore->copyIntoBuffer(0, nonstd::span(actualVertices.data(), actualVertices.size())); + SIMPLNX_RESULT_REQUIRE_VALID(verticesReadResult); + auto facesReadResult = facesStore->copyIntoBuffer(0, nonstd::span(actualFaces.data(), actualFaces.size())); + SIMPLNX_RESULT_REQUIRE_VALID(facesReadResult); + auto normalsReadResult = normalsStore->copyIntoBuffer(0, nonstd::span(actualNormals.data(), actualNormals.size())); + SIMPLNX_RESULT_REQUIRE_VALID(normalsReadResult); + CHECK(actualVertices == originalVertices); + CHECK(actualFaces == originalFaces); + + // The first three faces point along +Z, -Z, and +X. + // The sloped face has cross product (-2, -3, 6), with exact length 7. + // These literals preserve float-rounded components stored as double, which differ from double normalization. + const std::array expectedNormals = {0.0, 0.0, 1.0, 0.0, 0.0, -1.0, 1.0, 0.0, 0.0, -0.2857142984867095947265625, -0.4285714328289031982421875, 0.857142865657806396484375}; + for(usize valueIndex = 0; valueIndex < actualNormals.size(); ++valueIndex) + { + CAPTURE(valueIndex); + CHECK(actualNormals[valueIndex] == (preCancelled ? k_OutputSentinel : expectedNormals[valueIndex])); + } + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} diff --git a/test/H5IOTest.cpp b/test/H5IOTest.cpp index a916801835..2a43917fde 100644 --- a/test/H5IOTest.cpp +++ b/test/H5IOTest.cpp @@ -1,3 +1,5 @@ +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Uuid.hpp" #include "simplnx/UnitTest/HDF5DatasetProbe.hpp" #include "simplnx/Utilities/Parsing/HDF5/H5.hpp" #include "simplnx/Utilities/Parsing/HDF5/H5Support.hpp" @@ -5,17 +7,28 @@ #include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" #include "simplnx/Utilities/Parsing/HDF5/IO/GroupIO.hpp" +#include +#include +#include +#include + #include #include +#include #include #include #include #include #include +#include +#include #include +#include +#include #include +#include #include using namespace nx::core; @@ -392,3 +405,474 @@ TEST_CASE("HDF5 IO layer self-locks: concurrent open+query+read does not race", std::filesystem::remove(tmp); } + +namespace +{ +// The callback releases all HDF5 wrappers before the file cleanup checks run. +template +void WithHyperslabTestFile(BodyT&& body) +{ + const auto directory = std::filesystem::temp_directory_path() / ("h5io_hyperslab_" + Uuid::GenerateV4().str()); + const auto path = directory / "selection.h5"; + REQUIRE(std::filesystem::create_directory(directory)); + auto cleanup = MakeScopeGuard([&]() noexcept { + std::error_code error; + std::filesystem::remove(path, error); + std::filesystem::remove(directory, error); + }); + body(path); + CHECK(std::filesystem::remove(path)); + CHECK(std::filesystem::remove(directory)); +} + +void CreateHyperslabSentinelFile(const std::filesystem::path& path) +{ + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + auto outer = file.createGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.createGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.createDataset("values"); + const std::array sentinels{9, 9, 9, 9, 9, 9}; + REQUIRE(dataset.writeSpan({2, 3}, {sentinels.data(), sentinels.size()}).valid()); +} +} // namespace + +TEST_CASE("HDF5 hyperslab rejects malformed selections", "[simplnx][HDF5][hyperslab]") +{ + const auto caseIndex = GENERATE(0, 1); + DYNAMIC_SECTION((caseIndex == 0 ? "excess start rank" : "short logical span")) + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + CreateHyperslabSentinelFile(path); + auto file = HDF5::FileIO::AppendFile(path); + REQUIRE(file.isValid()); + auto outer = file.openGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.openGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.openDataset("values"); + REQUIRE(dataset.getId() > 0); + + // Both source elements exist even when the logical span contains only one. + const std::array source{4, 5}; + const std::vector start = caseIndex == 0 ? std::vector{1, 1, 0} : std::vector{1, 1}; + const usize spanLength = caseIndex == 0 ? source.size() : 1; + const auto result = dataset.writeSpanHyperslab({source.data(), spanLength}, start, {1, 2}); + CHECK(result.invalid()); + CHECK(dataset.readAsVector() == std::vector{9, 9, 9, 9, 9, 9}); + }); + } +} + +TEMPLATE_TEST_CASE("HDF5 hyperslab typed writes preserve unselected values", "[simplnx][HDF5][hyperslab]", int8, int16, int32, int64, uint8, uint16, uint32, uint64, float32, float64, bool) +{ + const usize inputCount = GENERATE(usize{2}, usize{3}); + DYNAMIC_SECTION("input values " << inputCount) + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + auto group = file.createGroup("typed"); + REQUIRE(group.isValid()); + auto dataset = group.createDataset("values"); + std::array sentinels{}; + std::array source{}; + std::array expected{}; + if constexpr(std::is_same_v) + { + sentinels = {true, true, true, true, true, true}; + source = {false, true, false}; + expected = {true, true, true, true, false, true}; + } + else + { + sentinels = {9, 9, 9, 9, 9, 9}; + source = {4, 5, 7}; + expected = {9, 9, 9, 9, 4, 5}; + } + REQUIRE(dataset.writeSpan({2, 3}, {sentinels.data(), sentinels.size()}).valid()); + REQUIRE(dataset.writeSpanHyperslab({source.data(), inputCount}, {1, 1}, {1, 2}).valid()); + std::array actual{}; + REQUIRE(dataset.readIntoSpan({actual.data(), actual.size()}).valid()); + CHECK(actual == expected); + }); + } +} + +namespace +{ +void CheckHyperslabErrorContext(const Result<>& result, const std::filesystem::path& path, int32 expectedCode) +{ + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + const auto& error = result.errors().front(); + CHECK(error.code == expectedCode); + CHECK(error.message.find("outer/inner/values") != std::string::npos); + CHECK(error.message.find(path.string()) != std::string::npos); +} + +// The caller resolves parentId before this helper takes the nonrecursive HDF5 lock. +void CreateSpecialHyperslabDataset(hid_t parentId, H5S_class_t spaceType, const std::vector& dimensions) +{ + std::lock_guard apiLock(HDF5::Support::ApiLock()); + const hid_t spaceId = spaceType == H5S_SIMPLE ? H5Screate_simple(static_cast(dimensions.size()), dimensions.data(), nullptr) : H5Screate(spaceType); + REQUIRE(spaceId >= 0); + auto spaceGuard = MakeScopeGuard([spaceId]() noexcept { H5Sclose(spaceId); }); + REQUIRE(H5Sget_simple_extent_type(spaceId) == spaceType); + const int expectedRank = spaceType == H5S_SIMPLE ? static_cast(dimensions.size()) : 0; + REQUIRE(H5Sget_simple_extent_ndims(spaceId) == expectedRank); + if(spaceType == H5S_SIMPLE) + { + std::vector actual(dimensions.size()); + REQUIRE(H5Sget_simple_extent_dims(spaceId, actual.data(), nullptr) == expectedRank); + REQUIRE(actual == dimensions); + } + const hid_t datasetId = H5Dcreate2(parentId, "values", H5T_NATIVE_INT32, spaceId, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + REQUIRE(datasetId >= 0); + auto datasetGuard = MakeScopeGuard([datasetId]() noexcept { H5Dclose(datasetId); }); +} +} // namespace + +TEST_CASE("HDF5 hyperslab rejects malformed selections after rank and size validation", "[simplnx][HDF5][hyperslab]") +{ + struct Selection + { + const char* name; + std::vector start; + std::vector count; + const char* diagnostic; + }; + const std::array selections{{ + {"empty start rank", {}, {1, 2}, "rank"}, + {"short start rank", {1}, {1, 2}, "rank"}, + {"empty count rank", {1, 1}, {}, "rank"}, + {"short count rank", {1, 1}, {1}, "rank"}, + {"excess count rank", {1, 1}, {1, 2, 1}, "rank"}, + {"both ranks empty", {}, {}, "rank"}, + {"start at nonempty boundary", {2, 0}, {1, 1}, "axis"}, + {"start beyond extent", {3, 0}, {1, 1}, "axis"}, + {"count beyond remaining extent", {1, 2}, {1, 2}, "axis"}, + {"maximum start", {std::numeric_limits::max(), 0}, {1, 1}, "axis"}, + {"maximum count", {0, 0}, {std::numeric_limits::max(), 1}, "axis"}, + {"empty first axis does not hide invalid second axis", {0, 4}, {0, 0}, "axis"}, + {"empty selection beyond both boundaries", {2, 4}, {0, 0}, "axis"}, + }}; + const usize caseIndex = GENERATE(Catch::Generators::range(usize{0}, usize{13})); + const auto& selection = selections[caseIndex]; + DYNAMIC_SECTION(selection.name) + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + CreateHyperslabSentinelFile(path); + auto file = HDF5::FileIO::AppendFile(path); + REQUIRE(file.isValid()); + auto outer = file.openGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.openGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.openDataset("values"); + REQUIRE(dataset.getId() > 0); + const std::array source{4, 5, 6, 7, 8, 9}; + const auto result = dataset.writeSpanHyperslab({source.data(), source.size()}, selection.start, selection.count); + CHECK(dataset.readAsVector() == std::vector{9, 9, 9, 9, 9, 9}); + CheckHyperslabErrorContext(result, path, -1012); + CHECK(result.errors().front().message.find(selection.diagnostic) != std::string::npos); + }); + } +} + +TEST_CASE("HDF5 hyperslab reports an empty input and a native write failure", "[simplnx][HDF5][hyperslab]") +{ + const bool readOnly = GENERATE(false, true); + DYNAMIC_SECTION((readOnly ? "native read-only failure" : "default-empty source span")) + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + CreateHyperslabSentinelFile(path); + auto file = readOnly ? HDF5::FileIO::ReadFile(path) : HDF5::FileIO::AppendFile(path); + REQUIRE(file.isValid()); + auto outer = file.openGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.openGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.openDataset("values"); + REQUIRE(dataset.getId() > 0); + const std::array source{4, 5}; + const nonstd::span input = readOnly ? nonstd::span{source.data(), source.size()} : nonstd::span{}; + const auto result = dataset.writeSpanHyperslab(input, {1, 1}, {1, 2}); + CHECK(dataset.readAsVector() == std::vector{9, 9, 9, 9, 9, 9}); + CheckHyperslabErrorContext(result, path, readOnly ? -1014 : -1012); + if(readOnly) + { + const auto& message = result.errors().front().message; + CHECK(message.find("H5Dwrite") != std::string::npos); + CHECK((message.find("status -1") != std::string::npos || message.find("error -1") != std::string::npos)); + } + }); + } +} + +TEST_CASE("HDF5 hyperslab empty and scalar selection contract", "[simplnx][HDF5][hyperslab]") +{ + SECTION("empty simple selection does not write, including read-only files") + { + const bool readOnly = GENERATE(false, true); + const usize caseIndex = GENERATE(usize{0}, usize{1}, usize{2}, usize{3}); + const std::array, 4> starts{{{0, 1}, {1, 0}, {0, 0}, {2, 3}}}; + const std::array, 4> counts{{{0, 2}, {1, 0}, {0, 0}, {0, 0}}}; + DYNAMIC_SECTION("read-only " << readOnly << ", selection " << caseIndex) + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + CreateHyperslabSentinelFile(path); + auto file = readOnly ? HDF5::FileIO::ReadFile(path) : HDF5::FileIO::AppendFile(path); + REQUIRE(file.isValid()); + auto outer = file.openGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.openGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.openDataset("values"); + REQUIRE(dataset.getId() > 0); + CHECK(dataset.writeSpanHyperslab({}, starts[caseIndex], counts[caseIndex]).valid()); + CHECK(dataset.readAsVector() == std::vector{9, 9, 9, 9, 9, 9}); + }); + } + } + + SECTION("zero extent permits only an in-bounds empty selection") + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + const hid_t fileId = file.getId(); + CreateSpecialHyperslabDataset(fileId, H5S_SIMPLE, {0, 3}); + auto dataset = file.openDataset("values"); + REQUIRE(dataset.getId() > 0); + CHECK(dataset.writeSpanHyperslab({}, {0, 1}, {0, 2}).valid()); + CHECK(dataset.writeSpanHyperslab({}, {1, 1}, {0, 2}).invalid()); + CHECK(dataset.getDimensions() == std::vector{0, 3}); + }); + } + + SECTION("zero count does not hide a rank mismatch") + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + CreateHyperslabSentinelFile(path); + auto file = HDF5::FileIO::AppendFile(path); + REQUIRE(file.isValid()); + auto outer = file.openGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.openGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.openDataset("values"); + REQUIRE(dataset.getId() > 0); + const auto result = dataset.writeSpanHyperslab({}, {0}, {0, 0}); + CHECK(dataset.readAsVector() == std::vector{9, 9, 9, 9, 9, 9}); + CheckHyperslabErrorContext(result, path, -1012); + CHECK(result.errors().front().message.find("rank") != std::string::npos); + }); + } + + SECTION("scalar hyperslabs fail while full scalar writes remain valid") + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + auto dataset = file.createDataset("values"); + const std::array sentinel{9}; + REQUIRE(dataset.writeSpan({}, {sentinel.data(), sentinel.size()}).valid()); + const hid_t datasetId = dataset.getId(); + REQUIRE(datasetId > 0); + { + std::lock_guard apiLock(HDF5::Support::ApiLock()); + const hid_t spaceId = H5Dget_space(datasetId); + REQUIRE(spaceId >= 0); + auto spaceGuard = MakeScopeGuard([spaceId]() noexcept { H5Sclose(spaceId); }); + REQUIRE(H5Sget_simple_extent_type(spaceId) == H5S_SCALAR); + } + const std::array source{4}; + const auto result = dataset.writeSpanHyperslab({source.data(), source.size()}, {}, {}); + CHECK(dataset.readAsVector() == std::vector{9}); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().message.find("scalar") != std::string::npos); + REQUIRE(dataset.writeSpan({}, {source.data(), source.size()}).valid()); + CHECK(dataset.readAsVector() == std::vector{4}); + }); + } + + SECTION("null dataspaces do not become empty simple selections") + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + const hid_t fileId = file.getId(); + CreateSpecialHyperslabDataset(fileId, H5S_NULL, {}); + auto dataset = file.openDataset("values"); + REQUIRE(dataset.getId() > 0); + const auto result = dataset.writeSpanHyperslab({}, {}, {}); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().message.find("null") != std::string::npos); + }); + } +} + +TEST_CASE("HDF5 hyperslab writes through an unlinked open dataset", "[simplnx][HDF5][hyperslab]") +{ + WithHyperslabTestFile([&](const std::filesystem::path& path) { + CreateHyperslabSentinelFile(path); + auto file = HDF5::FileIO::AppendFile(path); + REQUIRE(file.isValid()); + auto outer = file.openGroup("outer"); + REQUIRE(outer.isValid()); + auto inner = outer.openGroup("inner"); + REQUIRE(inner.isValid()); + auto dataset = inner.openDataset("values"); + const hid_t parentId = inner.getId(); + const hid_t datasetId = dataset.getId(); + REQUIRE(parentId > 0); + REQUIRE(datasetId > 0); + + // The open wrapper retains the dataset after its final link is removed. + { + std::lock_guard apiLock(HDF5::Support::ApiLock()); + REQUIRE(H5Ldelete(parentId, "values", H5P_DEFAULT) >= 0); + REQUIRE(H5Lexists(parentId, "values", H5P_DEFAULT) == 0); + REQUIRE(H5Iis_valid(datasetId) > 0); + REQUIRE(H5Iget_name(datasetId, nullptr, 0) == 0); + } + REQUIRE(dataset.isValid()); + + const std::array source{4, 5}; + Result<> writeResult; + REQUIRE_NOTHROW(writeResult = dataset.writeSpanHyperslab({source.data(), source.size()}, {1, 1}, {1, 2})); + REQUIRE(writeResult.valid()); + std::array actual{}; + { + std::lock_guard apiLock(HDF5::Support::ApiLock()); + REQUIRE(H5Dread(datasetId, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, actual.data()) >= 0); + } + CHECK(actual == std::array{9, 9, 9, 9, 4, 5}); + + const std::array rejectedSource{7, 8}; + Result<> rejectedResult; + REQUIRE_NOTHROW(rejectedResult = dataset.writeSpanHyperslab({rejectedSource.data(), rejectedSource.size()}, {1, 1, 0}, {1, 2})); + REQUIRE(rejectedResult.invalid()); + REQUIRE(rejectedResult.errors().size() == 1); + const auto& error = rejectedResult.errors().front(); + CHECK(error.code == -1012); + CHECK(error.message.find("values") != std::string::npos); + CHECK(error.message.find(path.string()) != std::string::npos); + CHECK(error.message.find("rank") != std::string::npos); + { + std::lock_guard apiLock(HDF5::Support::ApiLock()); + REQUIRE(H5Dread(datasetId, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, actual.data()) >= 0); + } + CHECK(actual == std::array{9, 9, 9, 9, 4, 5}); + }); +} + +namespace +{ +// Sparse creation must succeed before a test can claim to reach an overflow guard. +void CreateSparseHyperslabDataset(hid_t parentId, const std::vector& dimensions) +{ + const auto rank = static_cast(dimensions.size()); + const std::vector chunks(dimensions.size(), 1); + std::lock_guard apiLock(HDF5::Support::ApiLock()); + const hid_t spaceId = H5Screate_simple(rank, dimensions.data(), nullptr); + REQUIRE(spaceId >= 0); + auto spaceGuard = MakeScopeGuard([spaceId]() noexcept { H5Sclose(spaceId); }); + REQUIRE(H5Sget_simple_extent_type(spaceId) == H5S_SIMPLE); + REQUIRE(H5Sget_simple_extent_ndims(spaceId) == rank); + std::vector actual(dimensions.size()); + REQUIRE(H5Sget_simple_extent_dims(spaceId, actual.data(), nullptr) == rank); + REQUIRE(actual == dimensions); + const hid_t propertiesId = H5Pcreate(H5P_DATASET_CREATE); + REQUIRE(propertiesId >= 0); + auto propertiesGuard = MakeScopeGuard([propertiesId]() noexcept { H5Pclose(propertiesId); }); + REQUIRE(H5Pset_chunk(propertiesId, rank, chunks.data()) >= 0); + REQUIRE(H5Pset_alloc_time(propertiesId, H5D_ALLOC_TIME_LATE) >= 0); + REQUIRE(H5Pset_fill_time(propertiesId, H5D_FILL_TIME_NEVER) >= 0); + const hid_t datasetId = H5Dcreate2(parentId, "values", H5T_NATIVE_UINT8, spaceId, H5P_DEFAULT, propertiesId, H5P_DEFAULT); + REQUIRE(datasetId >= 0); + auto datasetGuard = MakeScopeGuard([datasetId]() noexcept { H5Dclose(datasetId); }); + REQUIRE(H5Dget_storage_size(datasetId) == 0); + const hid_t storedSpaceId = H5Dget_space(datasetId); + REQUIRE(storedSpaceId >= 0); + auto storedSpaceGuard = MakeScopeGuard([storedSpaceId]() noexcept { H5Sclose(storedSpaceId); }); + REQUIRE(H5Sget_simple_extent_type(storedSpaceId) == H5S_SIMPLE); + REQUIRE(H5Sget_simple_extent_ndims(storedSpaceId) == rank); + REQUIRE(H5Sget_simple_extent_dims(storedSpaceId, actual.data(), nullptr) == rank); + REQUIRE(actual == dimensions); +} + +void RequireSparseHyperslabStorage(hid_t datasetId) +{ + std::lock_guard apiLock(HDF5::Support::ApiLock()); + REQUIRE(H5Iis_valid(datasetId) > 0); + REQUIRE(H5Dget_storage_size(datasetId) == 0); +} +} // namespace + +TEST_CASE("HDF5 hyperslab sparse overflow fixture candidates", "[simplnx][HDF5][hyperslab][hyperslab-overflow-candidate]") +{ + const auto caseIndex = GENERATE(0, 1, 2); + DYNAMIC_SECTION("overflow fixture " << caseIndex) + { + WithHyperslabTestFile([&](const std::filesystem::path& path) { + { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + const hid_t fileId = file.getId(); + const std::vector byteOverflowDimensions = sizeof(usize) >= 8 ? std::vector{2147483648ULL, 2147483648ULL} : std::vector{65536, 8192}; + const std::vector dimensions = caseIndex == 0 ? std::vector{4294967296ULL, 4294967296ULL} : + caseIndex == 1 ? byteOverflowDimensions : + std::vector{4294967296ULL, 4294967296ULL, 0}; + std::vector nativeDimensions; + nativeDimensions.reserve(dimensions.size()); + for(const uint64 dimension : dimensions) + { + REQUIRE(dimension <= std::numeric_limits::max()); + nativeDimensions.push_back(static_cast(dimension)); + } + CreateSparseHyperslabDataset(fileId, nativeDimensions); + auto dataset = file.openDataset("values"); + const hid_t datasetId = dataset.getId(); + REQUIRE(datasetId > 0); + RequireSparseHyperslabStorage(datasetId); + const std::vector start(dimensions.size(), 0); + Result<> result; + if(caseIndex == 0) + { + const std::array input{4}; + result = dataset.writeSpanHyperslab({input.data(), input.size()}, start, dimensions); + } + else if(caseIndex == 1) + { + const std::array input{4}; + result = dataset.writeSpanHyperslab({input.data(), input.size()}, start, dimensions); + } + else + { + result = dataset.writeSpanHyperslab({}, start, dimensions); + } + RequireSparseHyperslabStorage(datasetId); + if(caseIndex == 2) + { + CHECK(result.valid()); + } + else + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors().front().code == -1012); + CHECK(result.errors().front().message.find(caseIndex == 0 ? "selected element count overflow" : "selected byte count overflow") != std::string::npos); + CHECK(result.errors().front().message.find(path.string()) != std::string::npos); + } + } + // A closed sparse fixture must remain a small metadata file. + CHECK(std::filesystem::file_size(path) < 1024ULL * 1024ULL); + }); + } +} diff --git a/test/IOFormat.cpp b/test/IOFormat.cpp index 2f1f169823..5c665c86c6 100644 --- a/test/IOFormat.cpp +++ b/test/IOFormat.cpp @@ -41,7 +41,7 @@ TEST_CASE("Memory Check", "IOTest") TEST_CASE("Data Format: ForceInCore keeps useOocData false and stores in memory", "[IOTest][DataFormat]") { auto app = Application::GetOrCreateInstance(); - UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceInCore, app->getPreferences()->valueAs(Preferences::k_LargeDataSize_Key)); + UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::ForceInCore, app->getPreferences()->value(Preferences::k_LargeDataSize_Key)); // ForceInCore is the only mode for which OOC is "not in use". REQUIRE_FALSE(app->getPreferences()->useOocData()); @@ -57,7 +57,7 @@ TEST_CASE("Data Format: ForceInCore keeps useOocData false and stores in memory" TEST_CASE("Data Format: Adaptive and ForceOutOfCore report useOocData true", "[IOTest][DataFormat]") { auto app = Application::GetOrCreateInstance(); - UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::Adaptive, app->getPreferences()->valueAs(Preferences::k_LargeDataSize_Key)); + UnitTest::PreferencesSentinel preferencesSentinel(DataStorageMode::Adaptive, app->getPreferences()->value(Preferences::k_LargeDataSize_Key)); // OOC is "in use" for both size-driven and always-out-of-core intents. REQUIRE(app->getPreferences()->useOocData()); diff --git a/test/IParallelAlgorithmTest.cpp b/test/IParallelAlgorithmTest.cpp index 772d401a48..13aca89141 100644 --- a/test/IParallelAlgorithmTest.cpp +++ b/test/IParallelAlgorithmTest.cpp @@ -1,6 +1,7 @@ #include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataStore.hpp" #include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/Utilities/ParallelDataAlgorithm.hpp" #include @@ -376,3 +377,137 @@ TEST_CASE("IParallelAlgorithm: requireStoresInMemory with nullptr entries keeps REQUIRE(algorithm.getParallelizationEnabled() == false); #endif } + +TEST_CASE("IParallelAlgorithm: storage requirements are cumulative", "[simplnx][IParallelAlgorithm]") +{ + DataStructure dataStructure; + auto inMemoryStore = std::make_shared>(ShapeType{10}, ShapeType{1}, 0.0f); + auto oocStore = std::make_shared>(ShapeType{10}, ShapeType{1}); + auto* inMemoryArray = DataArray::Create(dataStructure, "InMemoryArray", inMemoryStore); + auto* oocArray = DataArray::Create(dataStructure, "OocArray", oocStore); + REQUIRE(inMemoryArray != nullptr); + REQUIRE(oocArray != nullptr); + + SECTION("array requirements keep an earlier OOC restriction") + { + ParallelDataAlgorithm algorithm; + algorithm.requireArraysInMemory({oocArray}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + + algorithm.requireArraysInMemory({inMemoryArray}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + + algorithm.requireArraysInMemory({}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + + algorithm.requireArraysInMemory({nullptr}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + SECTION("array requirements disable work when OOC follows resident storage") + { + ParallelDataAlgorithm algorithm; + algorithm.requireArraysInMemory({inMemoryArray}); + algorithm.requireArraysInMemory({oocArray}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + SECTION("store requirements keep an earlier OOC restriction") + { + DataStore residentStore(ShapeType{10}, ShapeType{1}, 0.0f); + MockOocDataStore localOocStore(ShapeType{10}, ShapeType{1}); + + ParallelDataAlgorithm algorithm; + algorithm.requireStoresInMemory({&localOocStore}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + + algorithm.requireStoresInMemory({&residentStore}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + + algorithm.requireStoresInMemory({}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + + algorithm.requireStoresInMemory({nullptr}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + SECTION("store requirements disable work when OOC follows resident storage") + { + DataStore residentStore(ShapeType{10}, ShapeType{1}, 0.0f); + MockOocDataStore localOocStore(ShapeType{10}, ShapeType{1}); + + ParallelDataAlgorithm algorithm; + algorithm.requireStoresInMemory({&residentStore}); + algorithm.requireStoresInMemory({&localOocStore}); + REQUIRE(algorithm.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + SECTION("array and store requirements keep restrictions across helpers") + { + DataStore residentStore(ShapeType{10}, ShapeType{1}, 0.0f); + MockOocDataStore localOocStore(ShapeType{10}, ShapeType{1}); + + ParallelDataAlgorithm storeThenArray; + storeThenArray.requireStoresInMemory({&localOocStore}); + storeThenArray.requireArraysInMemory({inMemoryArray}); + REQUIRE(storeThenArray.getParallelizationEnabled() == false); + + ParallelDataAlgorithm arrayThenStore; + arrayThenStore.requireArraysInMemory({oocArray}); + arrayThenStore.requireStoresInMemory({&residentStore}); + REQUIRE(arrayThenStore.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + + SECTION("explicit disable remains in effect for array and store requirements") + { + DataStore residentStore(ShapeType{10}, ShapeType{1}, 0.0f); + + ParallelDataAlgorithm arrayAlgorithm; + arrayAlgorithm.setParallelizationEnabled(false); + arrayAlgorithm.requireArraysInMemory({inMemoryArray}); + REQUIRE(arrayAlgorithm.getParallelizationEnabled() == false); + arrayAlgorithm.requireArraysInMemory({}); + REQUIRE(arrayAlgorithm.getParallelizationEnabled() == false); + arrayAlgorithm.requireArraysInMemory({nullptr}); + REQUIRE(arrayAlgorithm.getParallelizationEnabled() == false); + + ParallelDataAlgorithm storeAlgorithm; + storeAlgorithm.setParallelizationEnabled(false); + storeAlgorithm.requireStoresInMemory({&residentStore}); + REQUIRE(storeAlgorithm.getParallelizationEnabled() == false); + storeAlgorithm.requireStoresInMemory({}); + REQUIRE(storeAlgorithm.getParallelizationEnabled() == false); + storeAlgorithm.requireStoresInMemory({nullptr}); + REQUIRE(storeAlgorithm.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } + +#ifdef SIMPLNX_ENABLE_MULTICORE + SECTION("explicit enable resets restrictions until another OOC requirement") + { + ParallelDataAlgorithm arrayAlgorithm; + arrayAlgorithm.requireArraysInMemory({oocArray}); + arrayAlgorithm.setParallelizationEnabled(true); + arrayAlgorithm.requireArraysInMemory({inMemoryArray}); + REQUIRE(arrayAlgorithm.getParallelizationEnabled() == true); + arrayAlgorithm.requireArraysInMemory({oocArray}); + REQUIRE(arrayAlgorithm.getParallelizationEnabled() == false); + + MockOocDataStore localOocStore(ShapeType{10}, ShapeType{1}); + DataStore residentStore(ShapeType{10}, ShapeType{1}, 0.0f); + ParallelDataAlgorithm storeAlgorithm; + storeAlgorithm.requireStoresInMemory({&localOocStore}); + storeAlgorithm.setParallelizationEnabled(true); + storeAlgorithm.requireStoresInMemory({&residentStore}); + REQUIRE(storeAlgorithm.getParallelizationEnabled() == true); + storeAlgorithm.requireStoresInMemory({&localOocStore}); + REQUIRE(storeAlgorithm.getParallelizationEnabled() == false); + UnitTest::CheckArraysInheritTupleDims(dataStructure); + } +#endif +} diff --git a/test/OocThresholdScopeTest.cpp b/test/OocThresholdScopeTest.cpp new file mode 100644 index 0000000000..480acf9fc0 --- /dev/null +++ b/test/OocThresholdScopeTest.cpp @@ -0,0 +1,166 @@ +#include "simplnx/Core/Application.hpp" +#include "simplnx/Core/Preferences.hpp" +#include "simplnx/UnitTest/AlgorithmTestScope.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include + +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +std::filesystem::path requireOwnedProfile() +{ + const char* rootText = std::getenv("SIMPLNX_P1_OWNED_ROOT"); + REQUIRE(rootText != nullptr); + const std::filesystem::path root(rootText); + REQUIRE(root.is_absolute()); +#if defined(_WIN32) + const auto expected = root / "AppData/Local/DREAM3DNX/preferences.json"; +#elif defined(__APPLE__) + const auto expected = root / "Library/Preferences/DREAM3DNX/preferences.json"; +#else + const auto expected = root / ".config/DREAM3DNX/preferences.json"; +#endif + REQUIRE(Preferences::DefaultFilePath("DREAM3DNX") == expected); + REQUIRE(Application::Instance() == nullptr); + return expected; +} + +void requireThreshold(const Preferences& preferences, bool present, nlohmann::json::value_t type, uint64 value) +{ + REQUIRE(preferences.contains(Preferences::k_LargeDataSize_Key) == present); + const auto actual = preferences.value(Preferences::k_LargeDataSize_Key); + REQUIRE(actual.type() == type); + REQUIRE(actual.is_number_integer()); + CHECK(actual.get() == value); +} +} // namespace + +TEST_CASE("P1 owned shared sentinel restores exact threshold and presence", "[.P1OwnedThreshold]") +{ + requireOwnedProfile(); + auto app = Application::GetOrCreateInstance(); + auto* preferences = app->getPreferences(); + preferences->removeValue(Preferences::k_LargeDataSize_Key); + preferences->removeValue(Preferences::k_DataStorageMode_Key); + + { + UnitTest::PreferencesSentinel sentinel(DataStorageMode::ForceInCore, nlohmann::json(uint64{0})); + REQUIRE(preferences->dataStorageMode() == DataStorageMode::ForceInCore); + requireThreshold(*preferences, true, nlohmann::json::value_t::number_unsigned, 0); + } + CHECK_FALSE(preferences->contains(Preferences::k_LargeDataSize_Key)); + CHECK_FALSE(preferences->contains(Preferences::k_DataStorageMode_Key)); + + preferences->setValue(Preferences::k_LargeDataSize_Key, std::numeric_limits::max()); + preferences->setDataStorageMode(DataStorageMode::Adaptive); + const auto originalMode = preferences->value(Preferences::k_DataStorageMode_Key); + { + UnitTest::PreferencesSentinel sentinel(DataStorageMode::ForceOutOfCore, nlohmann::json(uint64{17})); + requireThreshold(*preferences, true, nlohmann::json::value_t::number_unsigned, 17); + } + requireThreshold(*preferences, true, nlohmann::json::value_t::number_unsigned, std::numeric_limits::max()); + CHECK(preferences->value(Preferences::k_DataStorageMode_Key) == originalMode); + + CHECK_THROWS_AS(UnitTest::PreferencesSentinel(DataStorageMode::ForceInCore, int64{-1}), std::invalid_argument); + requireThreshold(*preferences, true, nlohmann::json::value_t::number_unsigned, std::numeric_limits::max()); + CHECK(preferences->value(Preferences::k_DataStorageMode_Key) == originalMode); + + for(const auto& value : {nlohmann::json(int64{37}), nlohmann::json(uint64{37})}) + { + preferences->setValue(Preferences::k_LargeDataSize_Key, value); + { + UnitTest::PreferencesSentinel sentinel(DataStorageMode::ForceInCore, nlohmann::json(uint64{0})); + } + requireThreshold(*preferences, true, value.type(), 37); + } + + app.reset(); + Application::DeleteInstance(); +} + +TEST_CASE("P1 owned public algorithm scope retains maximum threshold", "[.P1OwnedThreshold]") +{ + requireOwnedProfile(); + auto app = Application::GetOrCreateInstance(); + auto* preferences = app->getPreferences(); + preferences->setValue(Preferences::k_LargeDataSize_Key, std::numeric_limits::max()); + { + UnitTest::AlgorithmTestScope scope(UnitTest::AlgorithmTestScenario::InCoreAlgorithmOnInMemoryStore); + REQUIRE(preferences->dataStorageMode() == DataStorageMode::ForceInCore); + } + requireThreshold(*preferences, true, nlohmann::json::value_t::number_unsigned, std::numeric_limits::max()); + preferences->removeValue(Preferences::k_LargeDataSize_Key); + preferences->removeValue(Preferences::k_DataStorageMode_Key); + { + UnitTest::AlgorithmTestScope scope(UnitTest::AlgorithmTestScenario::InCoreAlgorithmOnInMemoryStore); + } + CHECK_FALSE(preferences->contains(Preferences::k_LargeDataSize_Key)); + CHECK_FALSE(preferences->contains(Preferences::k_DataStorageMode_Key)); + app.reset(); + Application::DeleteInstance(); +} + +TEST_CASE("P1 owned Application retains startup correction through clean reload", "[.P1OwnedThreshold]") +{ + const auto profile = requireOwnedProfile(); + std::filesystem::create_directories(profile.parent_path()); + { + std::ofstream stream(profile); + REQUIRE(stream.is_open()); + stream << "{\"large_data_size\":-1,\"large_datastructure_size\":18446744073709551615,\"plugins\":{\"P1\":{\"keep\":17}}}"; + } + Preferences freshDefaults; + const uint64 expectedDefault = freshDefaults.defaultValueAs(Preferences::k_LargeDataSize_Key); + auto app = Application::GetOrCreateInstance(); + auto* preferences = app->getPreferences(); + CHECK(preferences->valueAs(Preferences::k_LargeDataSize_Key) == expectedDefault); + const auto validStructure = preferences->value(Preferences::k_LargeDataStructureSize_Key); + REQUIRE(preferences->contains(Preferences::k_LargeDataStructureSize_Key)); + REQUIRE(validStructure.is_number_unsigned()); + // A legacy byte value loads as the nearest 0.01-GB step; UINT64_MAX becomes 17179869183.99 GB. + CHECK(validStructure.get() == uint64{18446744073698814198ULL}); + CHECK(preferences->pluginValue("P1", "keep") == nlohmann::json(17)); + + { + std::ofstream stream(profile); + REQUIRE(stream.is_open()); + stream << "{\"large_data_size\":0,\"large_datastructure_size\":18446744073709551615,\"plugins\":{\"P1\":{\"keep\":17}}}"; + } + const auto cleanLoad = app->loadPreferences(); + REQUIRE(cleanLoad.valid()); + CHECK(cleanLoad.warnings().empty()); + const auto first = app->takePreferenceLoadWarnings(); + REQUIRE(first.size() == 1); + CHECK(first.front().message.find("-1") != std::string::npos); + CHECK(first.front().message.find(std::to_string(expectedDefault)) != std::string::npos); + CHECK(app->takePreferenceLoadWarnings().empty()); + + { + std::ofstream stream(profile); + REQUIRE(stream.is_open()); + stream << "{\"large_datastructure_size\":\"bad\"}"; + } + REQUIRE(app->loadPreferences().valid()); + { + std::ofstream stream(profile); + REQUIRE(stream.is_open()); + stream << "{\"large_data_size\":-2}"; + } + REQUIRE(app->loadPreferences().valid()); + const auto ordered = app->takePreferenceLoadWarnings(); + REQUIRE(ordered.size() == 2); + CHECK(ordered[0].message.find("large_datastructure_size") != std::string::npos); + CHECK(ordered[1].message.find("large_data_size") != std::string::npos); + CHECK(app->takePreferenceLoadWarnings().empty()); + + app.reset(); + Application::DeleteInstance(); +} diff --git a/test/OwnedP1CatchCase.cmake b/test/OwnedP1CatchCase.cmake new file mode 100644 index 0000000000..ba1614cb94 --- /dev/null +++ b/test/OwnedP1CatchCase.cmake @@ -0,0 +1,42 @@ +if(NOT DEFINED P1_TEST_EXECUTABLE OR NOT EXISTS "${P1_TEST_EXECUTABLE}") + message(FATAL_ERROR "P1_TEST_EXECUTABLE must name an existing Catch executable") +endif() +if(NOT DEFINED P1_TEST_NAME OR P1_TEST_NAME STREQUAL "") + message(FATAL_ERROR "P1_TEST_NAME must select exactly one hidden Catch case") +endif() +if(NOT DEFINED P1_ROOT_BASE) + message(FATAL_ERROR "P1_ROOT_BASE was not provided") +endif() + +string(RANDOM LENGTH 12 ALPHABET 0123456789abcdef nonce) +set(case_root "${P1_ROOT_BASE}/p1-${nonce}") +file(MAKE_DIRECTORY "${case_root}/temp" "${case_root}/AppData/Local" "${case_root}/Library/Preferences" "${case_root}/.config") +file(REAL_PATH "${case_root}" case_root) + +# This CMake process is the CTest child. Its environment changes do not affect the developer process. +set(ENV{HOME} "${case_root}") +set(ENV{USERPROFILE} "${case_root}") +set(ENV{LOCALAPPDATA} "${case_root}/AppData/Local") +set(ENV{APPDATA} "${case_root}/AppData/Roaming") +set(ENV{TEMP} "${case_root}/temp") +set(ENV{TMP} "${case_root}/temp") +set(ENV{TMPDIR} "${case_root}/temp") +file(WRITE "${case_root}/.simplnx-test-profile" "${nonce}") +set(ENV{SIMPLNX_TEST_PROFILE_DIR} "${case_root}") +set(ENV{SIMPLNX_TEST_PROFILE_TOKEN} "${nonce}") +set(ENV{SIMPLNX_P1_OWNED_ROOT} "${case_root}") +unset(ENV{SIMPLNX_OOC_FAULT_INJECT}) + +execute_process( + COMMAND "${P1_TEST_EXECUTABLE}" "${P1_TEST_NAME}" --warn NoTests + WORKING_DIRECTORY "${case_root}" + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE error + TIMEOUT 120 +) +file(WRITE "${case_root}/stdout.txt" "${output}") +file(WRITE "${case_root}/stderr.txt" "${error}") +if(NOT result STREQUAL "0") + message(FATAL_ERROR "P1 owned case '${P1_TEST_NAME}' failed with '${result}'. Evidence: ${case_root}\n${output}\n${error}") +endif() diff --git a/test/ParallelChunkCodecTest.cpp b/test/ParallelChunkCodecTest.cpp index 6d0b772f16..04b8ab94c6 100644 --- a/test/ParallelChunkCodecTest.cpp +++ b/test/ParallelChunkCodecTest.cpp @@ -1,5 +1,6 @@ #include "simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp" #include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/UnitTest/CodecOwnershipCountersForTesting.hpp" #include "simplnx/Utilities/Parsing/HDF5/ChunkIndex.hpp" #include "simplnx/Utilities/Parsing/HDF5/ChunkShapePolicy.hpp" #include "simplnx/Utilities/Parsing/HDF5/DeflateEligibility.hpp" @@ -26,10 +27,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -896,6 +899,11 @@ TEST_CASE("ParallelChunkCodec skips deflate only for incompressible chunks", "[P std::vector readBack(source.size()); REQUIRE(H5Dread(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, readBack.data()) >= 0); REQUIRE(readBack == source); + std::vector boundedReadBack(source.size()); + const auto bounded = codec.readExtentIntoBufferBounded(Extent({0, 0, 0}, {1, k_ChunkBytes - 1, 0}), boundedReadBack, 1024 * 1024); + REQUIRE(bounded.valid()); + REQUIRE(bounded.value()); + REQUIRE(boundedReadBack == source); H5Dclose(dataset); H5Fclose(fileId); @@ -952,6 +960,11 @@ TEST_CASE("ParallelChunkCodec preserves rewritten skipped-deflate metadata for l // chunk with skipped-deflate bytes. The metadata filter mask is authoritative even // if the raw-read API reports a stale mask for this live chunk. REQUIRE(codec.inflateChunk(0) == source); + std::vector boundedReadBack(source.size()); + const auto bounded = codec.readExtentIntoBufferBounded(Extent({0, 0, 0}, {0, k_ChunkBytes - 1, 0}), boundedReadBack, 1024 * 1024); + REQUIRE(bounded.valid()); + REQUIRE(bounded.value()); + REQUIRE(boundedReadBack == source); } } @@ -2015,6 +2028,14 @@ TEST_CASE("H8 codecs reject inconsistent memory size and invalid memory IDs", "[ CHECK_FALSE(wrongSize.isEligible()); ParallelChunkCodec invalidType(path, "/data", {7, 5}, {3, 2}, {}, sizeof(int32), datasetId, H5I_INVALID_HID); CHECK_FALSE(invalidType.isEligible()); + std::array destination{}; + const Extent extent({0, 0}, {0, 0}); + const auto wrongBounded = wrongSize.readExtentIntoBufferBounded(extent, destination, 65536); + REQUIRE(wrongBounded.invalid()); + REQUIRE(wrongBounded.errors().front().code == BoundedRead::InvalidExtent); + const auto invalidBounded = invalidType.readExtentIntoBufferBounded(extent, destination, 65536); + REQUIRE(invalidBounded.invalid()); + REQUIRE(invalidBounded.errors().front().code == BoundedRead::MetadataFailure); } TEST_CASE("H8 codecs reject reduced precision and same-size structured file types", "[H8][ParallelChunkCodec]") @@ -2068,3 +2089,779 @@ TEST_CASE("H8 codecs reject reduced precision and same-size structured file type CHECK_FALSE(codec.isEligible()); // These cases test raw identity only. No scalar conversion from structured values is attempted. } + +TEST_CASE("ParallelChunkCodec preserves legacy stream acceptance", "[ParallelChunkCodec][ooc-review-codec-baseline]") +{ + /** + * @enum StreamCase + * @brief Selects complete and damaged compressed byte fixtures. + */ + enum class StreamCase + { + Complete, ///< Supplies the complete valid compressed image. + Trailing, ///< Adds bytes after a complete valid stream. + Checksum, ///< Changes the checksum of a complete stream. + Truncated, ///< Removes the final stored byte. + ShortOutput ///< Compresses fewer values than the nominal chunk requires. + }; + const auto streamCase = GENERATE(StreamCase::Complete, StreamCase::Trailing, StreamCase::Checksum, StreamCase::Truncated, StreamCase::ShortOutput); + CAPTURE(static_cast(streamCase)); + + const std::vector tupleShape{64}; + const std::vector componentShape{1}; + const std::vector expected(64, std::byte{37}); + const auto filePath = testFilePath("ParallelChunkCodec_legacy_stream_acceptance.h5"); + hid_t fileId = H5I_INVALID_HID; + hid_t datasetId = H5I_INVALID_HID; + createEmptyChunkedDeflateDataset(filePath, tupleShape, componentShape, tupleShape, H5T_NATIVE_UINT8, fileId, datasetId, 1); + auto fileGuard = MakeScopeGuard([&]() noexcept { H5Fclose(fileId); }); + auto datasetGuard = MakeScopeGuard([&]() noexcept { H5Dclose(datasetId); }); + + const uLong sourceBytes = streamCase == StreamCase::ShortOutput ? 63 : 64; + uLongf storedBytes = compressBound(sourceBytes); + std::vector payload(static_cast(storedBytes)); + REQUIRE(compress2(reinterpret_cast(payload.data()), &storedBytes, reinterpret_cast(expected.data()), sourceBytes, 1) == Z_OK); + payload.resize(static_cast(storedBytes)); + if(streamCase == StreamCase::Trailing) + { + payload.insert(payload.end(), {std::byte{0xD3}, std::byte{0x71}, std::byte{0xAA}}); + } + else if(streamCase == StreamCase::Checksum) + { + payload.back() ^= std::byte{1}; + } + else if(streamCase == StreamCase::Truncated) + { + payload.pop_back(); + } + const std::array offset{0, 0}; + REQUIRE(H5Dwrite_chunk(datasetId, H5P_DEFAULT, 0, offset.data(), payload.size(), payload.data()) >= 0); + REQUIRE(H5Fflush(fileId, H5F_SCOPE_LOCAL) >= 0); + + ParallelChunkCodec codec(filePath, k_DatasetName, tupleShape, tupleShape, componentShape, sizeof(uint8), datasetId, codecMemoryType()); + REQUIRE(codec.isEligible()); + std::vector boundedOutput(expected.size(), std::byte{0}); + const auto bounded = codec.readExtentIntoBufferBounded(Extent({0, 0}, {63, 0}), boundedOutput, 128 * 1024); + if(streamCase == StreamCase::Complete) + { + REQUIRE(bounded.valid()); + REQUIRE(bounded.value()); + REQUIRE(boundedOutput == expected); + } + else + { + REQUIRE(bounded.invalid()); + REQUIRE(bounded.errors().front().code == BoundedRead::DecodeFailure); + } + // The ordinary decoder retains its former trailing-byte acceptance. + if(streamCase == StreamCase::Complete || streamCase == StreamCase::Trailing) + { + CHECK(codec.inflateChunk(0) == expected); + } + else + { + CHECK_THROWS_AS(codec.inflateChunk(0), std::runtime_error); + } +} + +TEST_CASE("ParallelChunkCodec observes nominal edge and returned payload lifetimes", "[ParallelChunkCodec][ooc-review-codec-ownership]") +{ + const std::vector tuples{6, 5}; + const std::vector components{1}; + const std::vector chunks{4, 4}; + std::vector expected(30); + for(usize index = 0; index < expected.size(); ++index) + { + expected[index] = static_cast(index); + } + const auto path = testFilePath("ParallelChunkCodec_ownership.h5"); + createChunkedDeflateDataset(path, tuples, components, chunks, H5T_NATIVE_UINT8, expected); + const hid_t file = H5Fopen(path.string().c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + REQUIRE(file >= 0); + auto fileGuard = MakeScopeGuard([&]() noexcept { H5Fclose(file); }); + const hid_t dataset = H5Dopen2(file, k_DatasetName.c_str(), H5P_DEFAULT); + REQUIRE(dataset >= 0); + auto datasetGuard = MakeScopeGuard([&]() noexcept { H5Dclose(dataset); }); + ParallelChunkCodec codec(path, k_DatasetName, tuples, chunks, components, 1, dataset, codecMemoryType()); + REQUIRE(codec.isEligible()); + CodecOwnershipCountersForTesting counters; + const auto before = GetCodecOwnershipStateForTesting(); + REQUIRE(before.outstandingBuffers == 0); + REQUIRE(before.activeTasks == 0); + const auto previous = SetCodecIoObserverForTesting({&counters, &CodecOwnershipCountersForTesting::Observe}); + auto observerGuard = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const std::array selected{0, 1, 2, 3}; + std::vector result(expected.size()); + codec.inflateChunksIntoSpan(result, selected); + REQUIRE(result == expected); + const auto after = GetCodecOwnershipStateForTesting(); + CHECK(after.outstandingBuffers == 0); + CHECK(after.activeTasks == 0); + CHECK(after.integrityFailures == before.integrityFailures); + CHECK(after.unobservedReturns == before.unobservedReturns); + CHECK(counters.nominalAcquired.load() == 4); + CHECK(counters.edgeAcquired.load() == 3); + CHECK(counters.live.load() == 0); + CHECK_FALSE(counters.invalidRelease.load()); + CHECK(counters.peak.load() >= 18); +} + +TEST_CASE("ParallelChunkCodec observation refuses active replacement", "[ParallelChunkCodec][ooc-review-codec-ownership]") +{ + CodecOwnershipCountersForTesting original; + CodecOwnershipCountersForTesting replacement; + const auto before = GetCodecOwnershipStateForTesting(); + REQUIRE(before.outstandingBuffers == 0); + REQUIRE(before.activeTasks == 0); + const auto previous = SetCodecIoObserverForTesting({&original, &CodecOwnershipCountersForTesting::Observe}); + auto observerGuard = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const auto path = testFilePath("ownership-control.h5"); + { + LegacyPayloadTaskScopeForTesting task; + (void)SetCodecIoObserverForTesting({&replacement, &CodecOwnershipCountersForTesting::Observe}); + LegacyPayloadTicketForTesting ticket; + const std::vector bytes(64); + ticket.acquire(path, "/data", bytes, LegacyPayloadKindForTesting::Nominal); + } + const auto after = GetCodecOwnershipStateForTesting(); + CHECK(after.integrityFailures == before.integrityFailures + 1); + CHECK(after.outstandingBuffers == 0); + CHECK(after.activeTasks == 0); + CHECK(original.nominalAcquired.load() == 1); + CHECK(replacement.nominalAcquired.load() == 0); + CHECK(original.live.load() == 0); +} + +TEST_CASE("ParallelChunkCodec checks stranded receipts on each worker exit", "[ParallelChunkCodec][ooc-review-codec-ownership]") +{ + CodecOwnershipCountersForTesting counters; + const auto before = GetCodecOwnershipStateForTesting(); + REQUIRE(before.outstandingBuffers == 0); + REQUIRE(before.activeTasks == 0); + const auto previous = SetCodecIoObserverForTesting({&counters, &CodecOwnershipCountersForTesting::Observe}); + auto observerGuard = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const auto path = testFilePath("ownership-worker-control.h5"); + const std::array selected{0, 1}; + const auto loader = [](uint64 index) { return index; }; + const auto sink = [&](usize index, uint64&& value) { + (void)value; + LegacyPayloadTicketForTesting ticket; + const std::vector bytes(64); + ticket.acquire(path, "/data", bytes, LegacyPayloadKindForTesting::Returned); + ticket.forwardReturn(); + if(index == 0) + { + throw std::runtime_error("receipt worker control"); + } + }; + CHECK_THROWS_AS(ParallelLoadChunks(selected, loader, sink), std::runtime_error); + const auto after = GetCodecOwnershipStateForTesting(); + // Each task strands one receipt. The throwing task also reaches error aggregation with that charge. + CHECK(after.errorAggregations == before.errorAggregations + 1); + CHECK(after.integrityFailures == before.integrityFailures + selected.size() + 1); + CHECK(after.outstandingBuffers == 0); + CHECK(after.activeTasks == 0); + CHECK(counters.live.load() == 0); + CHECK_FALSE(counters.invalidRelease.load()); +} + +TEST_CASE("ParallelChunkCodec releases a throwing vector sink before error aggregation", "[ParallelChunkCodec][ooc-review-codec-ownership]") +{ + CodecOwnershipCountersForTesting counters; + const auto before = GetCodecOwnershipStateForTesting(); + REQUIRE(before.outstandingBuffers == 0); + REQUIRE(before.activeTasks == 0); + const auto previous = SetCodecIoObserverForTesting({&counters, &CodecOwnershipCountersForTesting::Observe}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const auto path = testFilePath("throwing-vector-sink-control.h5"); + const std::array selected{0}; + const auto loader = [&](uint64) { + LegacyPayloadTicketForTesting ticket; + std::vector bytes(64); + ticket.acquire(path, "/data", bytes, LegacyPayloadKindForTesting::Returned); + ticket.forwardReturn(); + return bytes; + }; + const auto sink = [](usize, std::vector&&) { throw std::runtime_error("throwing vector sink"); }; + CHECK_THROWS_AS(ParallelLoadChunks>(selected, loader, sink), std::runtime_error); + const auto after = GetCodecOwnershipStateForTesting(); + REQUIRE(after.errorAggregations == before.errorAggregations + 1); + REQUIRE(after.integrityFailures == before.integrityFailures); + REQUIRE(after.outstandingBuffers == 0); + REQUIRE(after.activeTasks == 0); + REQUIRE(counters.live.load() == 0); +} + +TEST_CASE("ParallelChunkCodec observes simultaneous worker payloads deterministically", "[ParallelChunkCodec][ooc-review-codec-ownership]") +{ + CodecOwnershipCountersForTesting counters; + const auto before = GetCodecOwnershipStateForTesting(); + REQUIRE(before.outstandingBuffers == 0); + REQUIRE(before.activeTasks == 0); + const auto previous = SetCodecIoObserverForTesting({&counters, &CodecOwnershipCountersForTesting::Observe}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const auto path = testFilePath("simultaneous-ownership-control.h5"); + std::latch acquired(2); + std::latch release(1); + std::mutex failureMutex; + std::exception_ptr failure; + std::thread first; + std::thread second; + bool released = false; + auto joined = MakeScopeGuard([&]() noexcept { + if(!released) + { + release.count_down(); + } + if(first.joinable()) + { + first.join(); + } + if(second.joinable()) + { + second.join(); + } + }); + const auto worker = [&]() { + bool signaled = false; + try + { + LegacyPayloadTaskScopeForTesting task; + LegacyPayloadTicketForTesting ticket; + std::vector bytes(64); + ticket.acquire(path, "/data", bytes, LegacyPayloadKindForTesting::Returned); + acquired.count_down(); + signaled = true; + release.wait(); + } catch(...) + { + { + std::lock_guard lock(failureMutex); + if(!failure) + { + failure = std::current_exception(); + } + } + if(!signaled) + { + acquired.count_down(); + } + } + }; + first = std::thread(worker); + second = std::thread(worker); + acquired.wait(); + { + std::lock_guard lock(failureMutex); + REQUIRE(failure == nullptr); + } + REQUIRE(counters.live.load() == 128); + REQUIRE(counters.peak.load() == 128); + released = true; + release.count_down(); + first.join(); + second.join(); + const auto after = GetCodecOwnershipStateForTesting(); + REQUIRE(after.outstandingBuffers == 0); + REQUIRE(after.activeTasks == 0); + REQUIRE(after.integrityFailures == before.integrityFailures); + REQUIRE(counters.live.load() == 0); +} + +TEST_CASE("ParallelChunkCodec labels a non-codec vector return as unobserved", "[ParallelChunkCodec][ooc-review-codec-ownership]") +{ + CodecOwnershipCountersForTesting counters; + const auto before = GetCodecOwnershipStateForTesting(); + REQUIRE(before.outstandingBuffers == 0); + REQUIRE(before.activeTasks == 0); + const auto previous = SetCodecIoObserverForTesting({&counters, &CodecOwnershipCountersForTesting::Observe}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const std::array selected{0}; + const auto loader = [](uint64) { return std::vector(64); }; + const auto sink = [](usize, std::vector&& bytes) { REQUIRE(bytes.size() == 64); }; + ParallelLoadChunks>(selected, loader, sink); + const auto after = GetCodecOwnershipStateForTesting(); + REQUIRE(after.unobservedReturns == before.unobservedReturns + 1); + REQUIRE(after.integrityFailures == before.integrityFailures); + REQUIRE(after.outstandingBuffers == 0); + REQUIRE(after.activeTasks == 0); +} + +namespace +{ +/** @brief Measures actual bounded allocations while preserving the existing scoped observer. */ +struct BoundedCodecObservation +{ + uint64 live = 0; + uint64 peak = 0; + uint64 acquired = 0; + uint64 released = 0; + uint64 rawReads = 0; + uint64 decoded = 0; + uint64 queries = 0; + uint64 metadataCalls = 0; + uint64 terminalRequests = 0; + bool invalid = false; + bool inject = false; + BoundedFaultPointForTesting fault = BoundedFaultPointForTesting::MetadataQuery; + + static void Observe(void* context, const fs::path&, std::string_view, CodecIoEventForTesting event, uint64 input, uint64 output) noexcept + { + auto& self = *static_cast(context); + if(event == CodecIoEventForTesting::BoundedAcquired) + { + self.acquired += input; + self.live += input; + self.peak = std::max(self.peak, self.live); + } + else if(event == CodecIoEventForTesting::BoundedReleased) + { + if(input > self.live) + { + self.invalid = true; + } + else + { + self.live -= input; + self.released += input; + } + } + else if(event == CodecIoEventForTesting::DiagnosticRequested) + { + ++self.terminalRequests; + } + else if(event == CodecIoEventForTesting::DiagnosticRetained) + { + if(self.live != 0 || input + output > BoundedRead::k_DiagnosticBytes) + { + self.invalid = true; + } + } + else if(event == CodecIoEventForTesting::StoredRead) + { + ++self.rawReads; + } + else if(event == CodecIoEventForTesting::Inflated) + { + self.decoded += input; + } + else if(event == CodecIoEventForTesting::ChunkRecordQuery) + { + ++self.queries; + } + else if(event >= CodecIoEventForTesting::MetadataTypeCall && event <= CodecIoEventForTesting::MetadataSpaceCall) + { + ++self.metadataCalls; + } + } + + static bool Fault(void* context, const fs::path&, std::string_view, BoundedFaultPointForTesting point) noexcept + { + const auto& self = *static_cast(context); + return self.inject && self.fault == point; + } +}; +} // namespace + +TEST_CASE("Bounded codec reads strided physical component-split chunks within actual scratch", "[ParallelChunkCodec][bounded]") +{ + const int filter = GENERATE(0, 1); + const fs::path path = testFilePath("bounded_component_split_" + std::to_string(filter) + ".h5"); + const std::array shape{3, 5, 3}; + const std::array physicalChunks{2, 4, 2}; + std::vector values(45); + std::iota(values.begin(), values.end(), uint16{0}); + hid_t file = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t copiedType = H5I_INVALID_HID; + { + std::lock_guard lock(Support::ApiLock()); + file = H5Fcreate(path.string().c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + REQUIRE(file >= 0); + const hid_t space = H5Screate_simple(3, shape.data(), nullptr); + const hid_t properties = H5Pcreate(H5P_DATASET_CREATE); + REQUIRE(space >= 0); + REQUIRE(properties >= 0); + REQUIRE(H5Pset_chunk(properties, 3, physicalChunks.data()) >= 0); + if(filter != 0) + { + REQUIRE(H5Pset_deflate(properties, 1) >= 0); + } + dataset = H5Dcreate2(file, "data", H5T_NATIVE_UINT16, space, H5P_DEFAULT, properties, H5P_DEFAULT); + REQUIRE(dataset >= 0); + REQUIRE(H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, values.data()) >= 0); + REQUIRE(H5Fflush(file, H5F_SCOPE_LOCAL) >= 0); + copiedType = H5Tcopy(H5T_NATIVE_UINT16); + REQUIRE(copiedType >= 0); + H5Pclose(properties); + H5Sclose(space); + } + auto cleanup = MakeScopeGuard([&]() noexcept { + std::lock_guard lock(Support::ApiLock()); + H5Dclose(dataset); + H5Fclose(file); + }); + ParallelChunkCodec codec(path, "data", {3, 5}, {2, 4}, {3}, sizeof(uint16), dataset, copiedType); + { + std::lock_guard lock(Support::ApiLock()); + H5Tclose(copiedType); + } + // The bounded path uses the actual three-dimensional physical chunk shape. + const Extent extent({0, 1, 0}, {2, 4, 2}, {2, 2, 1}); + std::vector output(12, uint16{0xCAFE}); + std::vector expected; + for(uint64 z : {0ULL, 2ULL}) + { + for(uint64 y : {1ULL, 3ULL}) + { + for(uint64 component = 0; component < 3; ++component) + { + expected.push_back(values[(z * 5 + y) * 3 + component]); + } + } + } + BoundedCodecObservation observation; + const auto previous = SetCodecIoObserverForTesting({&observation, &BoundedCodecObservation::Observe, &BoundedCodecObservation::Fault}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const auto destination = nonstd::span(reinterpret_cast(output.data()), output.size() * sizeof(uint16)); + SECTION("checked metadata returns empty invalid state before work on an unreviewed build") + { + REQUIRE(BoundedRead::DiagnosticRepresentationSupported()); + REQUIRE(BoundedRead::EmptyInvalidResultControlBytes() >= 2 * sizeof(Result)); + observation.inject = true; + observation.fault = BoundedFaultPointForTesting::CheckedMetadataUnsupportedBuild; + const std::array origin{0, 0, 0}; + auto result = codec.getChunkInfoChecked(origin); + REQUIRE(result.invalid()); + REQUIRE(result.errors().empty()); + REQUIRE(result.errors().capacity() == 0); + REQUIRE(result.warnings().empty()); + REQUIRE(result.warnings().capacity() == 0); + auto moved = std::move(result); + REQUIRE(moved.invalid()); + REQUIRE(moved.errors().empty()); + REQUIRE(moved.warnings().empty()); + REQUIRE(observation.queries == 0); + REQUIRE(observation.metadataCalls == 0); + REQUIRE(observation.terminalRequests == 0); + REQUIRE(observation.rawReads == 0); + REQUIRE(observation.acquired == 0); + REQUIRE(output == std::vector(12, uint16{0xCAFE})); + } + SECTION("subminimum and metadata refusal preserve the sentinel before any query") + { + for(const uint64 allowance : std::array{0, BoundedRead::k_DiagnosticBytes - 1}) + { + auto result = codec.readExtentIntoBufferBounded(extent, destination, allowance); + REQUIRE(result.valid()); + REQUIRE_FALSE(result.value()); + REQUIRE(result.warnings().empty()); + } + auto result = codec.readExtentIntoBufferBounded(extent, destination, BoundedRead::k_DiagnosticBytes); + REQUIRE(result.valid()); + REQUIRE_FALSE(result.value()); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings().front().code == BoundedRead::Insufficient); + REQUIRE(BoundedRead::ResultCapacityBytes(result) <= BoundedRead::k_DiagnosticBytes); + REQUIRE(observation.queries == 0); + REQUIRE(observation.rawReads == 0); + REQUIRE(output == std::vector(12, uint16{0xCAFE})); + } + SECTION("complete selected values and actual allocation lifetime") + { + constexpr uint64 allowance = 1024 * 1024; + auto result = codec.readExtentIntoBufferBounded(extent, destination, allowance); + REQUIRE(result.valid()); + REQUIRE(result.value()); + REQUIRE(output == expected); + REQUIRE(observation.rawReads > 0); + REQUIRE(observation.peak <= allowance - BoundedRead::k_DiagnosticBytes); + REQUIRE(observation.queries >= 2 * observation.rawReads); + } + SECTION("native tag independently rejects the wrong element size") + { + ParallelChunkCodec wrongSize(path, "data", {3, 5}, {2, 4}, {3}, 1, dataset, codecMemoryType()); + auto result = wrongSize.readExtentIntoBufferBounded(extent, destination, 1024 * 1024); + REQUIRE(result.invalid()); + REQUIRE(result.errors().front().code == BoundedRead::InvalidExtent); + REQUIRE(observation.rawReads == 0); + REQUIRE(output == std::vector(12, uint16{0xCAFE})); + } + SECTION("faults do not fall back to a typed filtered read") + { + observation.inject = true; + const auto fault = GENERATE(BoundedFaultPointForTesting::NativeSizeQuery, BoundedFaultPointForTesting::ContradictoryRecord, BoundedFaultPointForTesting::MetadataQuery, + BoundedFaultPointForTesting::SecondPassLargerRecord, BoundedFaultPointForTesting::StoredAllocation, BoundedFaultPointForTesting::RawRead); + observation.fault = fault; + auto result = codec.readExtentIntoBufferBounded(extent, destination, 1024 * 1024); + REQUIRE(result.invalid()); + if(fault == BoundedFaultPointForTesting::NativeSizeQuery || fault == BoundedFaultPointForTesting::ContradictoryRecord || fault == BoundedFaultPointForTesting::MetadataQuery) + { + REQUIRE(result.errors().front().code == BoundedRead::MetadataFailure); + } + REQUIRE(observation.rawReads == 0); + REQUIRE(output == std::vector(12, uint16{0xCAFE})); + } + SECTION("decoder window admission refuses before reading payload") + { + if(filter == 1) + { + const auto result = codec.readExtentIntoBufferBounded(extent, destination, 32768); + REQUIRE(result.valid()); + REQUIRE_FALSE(result.value()); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings().front().code == BoundedRead::Insufficient); + REQUIRE(observation.queries > 0); + REQUIRE(observation.rawReads == 0); + REQUIRE(output == std::vector(12, uint16{0xCAFE})); + } + } + SECTION("a real zlib window request refuses an unexpected post-read quota reduction") + { + if(filter == 1) + { + observation.inject = true; + observation.fault = BoundedFaultPointForTesting::DecoderQuotaAfterPayload; + const auto result = codec.readExtentIntoBufferBounded(extent, destination, 1024 * 1024); + REQUIRE(result.invalid()); + REQUIRE(result.errors().front().code == BoundedRead::DecoderQuotaFailure); + REQUIRE(observation.rawReads > 0); + } + } + SECTION("deflate allocation failure destroys initialized scratch") + { + if(filter == 1) + { + observation.inject = true; + observation.fault = GENERATE(BoundedFaultPointForTesting::NominalAllocation, BoundedFaultPointForTesting::DecoderAllocation); + auto result = codec.readExtentIntoBufferBounded(extent, destination, 1024 * 1024); + REQUIRE(result.invalid()); + REQUIRE(result.errors().front().code == BoundedRead::AllocationFailure); + REQUIRE(observation.rawReads == 0); + } + } + REQUIRE_FALSE(observation.invalid); + REQUIRE(observation.live == 0); + REQUIRE(observation.acquired == observation.released); +} + +TEST_CASE("Bounded codec distinguishes sparse fill from metadata failure", "[ParallelChunkCodec][bounded]") +{ + const int fillChoice = GENERATE(0, 1, 2); + const uint16 fillValue = 123; + const fs::path path = testFilePath("bounded_sparse_fill.h5"); + hid_t file = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + { + std::lock_guard lock(Support::ApiLock()); + file = H5Fcreate(path.string().c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + REQUIRE(file >= 0); + const std::array dimensions{2, 3, 1}; + const std::array chunks{1, 2, 1}; + const hid_t space = H5Screate_simple(3, dimensions.data(), nullptr); + const hid_t properties = H5Pcreate(H5P_DATASET_CREATE); + REQUIRE(space >= 0); + REQUIRE(properties >= 0); + REQUIRE(H5Pset_chunk(properties, 3, chunks.data()) >= 0); + REQUIRE(H5Pset_deflate(properties, 1) >= 0); + if(fillChoice != 0) + { + REQUIRE(H5Pset_fill_value(properties, H5T_NATIVE_UINT16, fillChoice == 1 ? &fillValue : nullptr) >= 0); + } + dataset = H5Dcreate2(file, "data", H5T_NATIVE_UINT16, space, H5P_DEFAULT, properties, H5P_DEFAULT); + REQUIRE(dataset >= 0); + H5Pclose(properties); + H5Sclose(space); + } + auto cleanup = MakeScopeGuard([&]() noexcept { + std::lock_guard lock(Support::ApiLock()); + H5Dclose(dataset); + H5Fclose(file); + }); + ParallelChunkCodec codec(path, "data", {2, 3}, {1, 2}, {1}, sizeof(uint16), dataset, codecMemoryType()); + std::vector output(6, 77); + BoundedCodecObservation observation; + const auto previous = SetCodecIoObserverForTesting({&observation, &BoundedCodecObservation::Observe, &BoundedCodecObservation::Fault}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + const auto result = codec.readExtentIntoBufferBounded(Extent({0, 0, 0}, {1, 2, 0}), nonstd::span(reinterpret_cast(output.data()), output.size() * sizeof(uint16)), 16384); + REQUIRE(result.valid()); + if(fillChoice == 2) + { + REQUIRE_FALSE(result.value()); + REQUIRE(result.warnings().size() == 1); + REQUIRE(result.warnings().front().code == BoundedRead::UndefinedFill); + REQUIRE(output == std::vector(6, 77)); + } + else + { + REQUIRE(result.value()); + REQUIRE(output == std::vector(6, fillChoice == 1 ? fillValue : 0)); + } + REQUIRE(observation.rawReads == 0); + REQUIRE(observation.decoded == 0); + REQUIRE(observation.live == 0); + const std::array origin{0, 0, 0}; + const auto info = codec.getChunkInfoChecked(origin); + REQUIRE(info.valid()); + REQUIRE_FALSE(info.value().allocated); + REQUIRE(info.value().storedSize == 0); +} + +TEST_CASE("Bounded codec rejects a second-pass request for an unprepared decoder", "[ParallelChunkCodec][bounded]") +{ + const fs::path path = testFilePath("bounded_unprepared_decoder.h5"); + hid_t file = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + std::array raw{}; + { + std::lock_guard lock(Support::ApiLock()); + createEmptyChunkedDeflateDataset(path, {64}, {1}, {64}, H5T_NATIVE_UINT8, file, dataset, 1); + const std::array origin{0, 0}; + REQUIRE(H5Dwrite_chunk(dataset, H5P_DEFAULT, 1, origin.data(), raw.size(), raw.data()) >= 0); + REQUIRE(H5Fflush(file, H5F_SCOPE_LOCAL) >= 0); + } + auto cleanup = MakeScopeGuard([&]() noexcept { + std::lock_guard lock(Support::ApiLock()); + H5Dclose(dataset); + H5Fclose(file); + }); + ParallelChunkCodec codec(path, "data", {64}, {64}, {1}, sizeof(uint8), dataset, codecMemoryType()); + BoundedCodecObservation observation; + observation.inject = true; + observation.fault = BoundedFaultPointForTesting::SecondPassUnpreparedDecoder; + const auto previous = SetCodecIoObserverForTesting({&observation, &BoundedCodecObservation::Observe, &BoundedCodecObservation::Fault}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + std::array destination; + destination.fill(std::byte{0x7B}); + const auto result = codec.readExtentIntoBufferBounded(Extent({0, 0}, {63, 0}), destination, 65536); + REQUIRE(result.invalid()); + REQUIRE(result.errors().front().code == BoundedRead::MetadataChanged); + REQUIRE(observation.rawReads == 0); + REQUIRE(std::all_of(destination.begin(), destination.end(), [](std::byte value) { return value == std::byte{0x7B}; })); + REQUIRE(observation.live == 0); + REQUIRE_FALSE(observation.invalid); +} + +TEST_CASE("Bounded codec scatter preserves selected values across contiguous runs and gaps", "[ParallelChunkCodec][bounded][bounded-scatter]") +{ + struct Fixture + { + const char* name; + std::vector tupleShape; + std::vector componentShape; + std::vector physicalChunks; + Extent extent; + std::vector expected; + bool sparse = false; + }; + // Literal outputs select from a dense row-major sequence that starts at 101. + // These expectations do not use the reader's strides, chunk intersections or scatter helpers. + const std::vector fixtures = { + {"full trailing suffix", {2, 3, 4}, {1}, {1, 3, 4, 1}, Extent({0, 0, 0, 0}, {1, 2, 3, 0}), {101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124}}, + {"nonzero chunk and extent origins", {4, 3, 4}, {1}, {2, 3, 4, 1}, Extent({2, 1, 0, 0}, {3, 2, 3, 0}), {129, 130, 131, 132, 133, 134, 135, 136, 141, 142, 143, 144, 145, 146, 147, 148}}, + {"adjacent source with output row gaps", {2, 4}, {1}, {2, 2, 1}, Extent({0, 0, 0}, {1, 3, 0}), {101, 102, 103, 104, 105, 106, 107, 108}}, + {"adjacent output with nominal source gaps", {2, 4}, {1}, {2, 4, 1}, Extent({0, 1, 0}, {1, 2, 0}), {102, 103, 106, 107}}, + {"physical component splits", {2, 3}, {3}, {2, 3, 2}, Extent({0, 0, 0}, {1, 2, 2}), {101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118}}, + {"clamped physical edges retain nominal padding", {3, 5}, {1}, {2, 4, 1}, Extent({1, 2, 0}, {2, 4, 0}), {108, 109, 110, 113, 114, 115}}, + {"tuple and component strides", {3, 4}, {3}, {2, 3, 2}, Extent({0, 1, 0}, {2, 3, 2}, {2, 2, 2}), {104, 106, 110, 112, 128, 130, 134, 136}}, + {"rank one contiguous", {9}, {}, {4}, Extent({1}, {8}), {102, 103, 104, 105, 106, 107, 108, 109}}, + {"rank one strided", {9}, {}, {4}, Extent({1}, {8}, {3}), {102, 105, 108}}, + {"singleton axes inside a contiguous suffix", {2, 1, 3, 1}, {1}, {2, 1, 3, 1, 1}, Extent({0, 0, 0, 0, 0}, {1, 0, 2, 0, 0}), {101, 102, 103, 104, 105, 106}}, + {"single selected scalar", {1}, {1}, {1, 1}, Extent({0, 0}, {0, 0}), {101}}, + {"sparse fill stays scalar", {2, 4}, {1}, {2, 2, 1}, Extent({0, 0, 0}, {1, 3, 0}), {513, 513, 513, 513, 513, 513, 513, 513}, true}}; + const int filter = GENERATE(0, 1); + for(const auto& fixture : fixtures) + { + DYNAMIC_SECTION(fixture.name << ", deflate=" << filter) + { + const fs::path path = testFilePath("bounded_scatter_" + std::to_string(filter) + ".h5"); + std::vector shape(fixture.tupleShape.begin(), fixture.tupleShape.end()); + shape.insert(shape.end(), fixture.componentShape.begin(), fixture.componentShape.end()); + REQUIRE(shape.size() == fixture.physicalChunks.size()); + std::vector values(product(fixture.tupleShape) * product(fixture.componentShape)); + std::iota(values.begin(), values.end(), uint16{101}); + const uint16 fillValue = fixture.sparse ? uint16{513} : uint16{0x4F4F}; + hid_t file = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t properties = H5I_INVALID_HID; + auto cleanup = MakeScopeGuard([&]() noexcept { + std::lock_guard lock(Support::ApiLock()); + if(dataset >= 0) + { + H5Dclose(dataset); + } + if(properties >= 0) + { + H5Pclose(properties); + } + if(space >= 0) + { + H5Sclose(space); + } + if(file >= 0) + { + H5Fclose(file); + } + }); + { + std::lock_guard lock(Support::ApiLock()); + file = H5Fcreate(path.string().c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + REQUIRE(file >= 0); + space = H5Screate_simple(static_cast(shape.size()), shape.data(), nullptr); + REQUIRE(space >= 0); + properties = H5Pcreate(H5P_DATASET_CREATE); + REQUIRE(properties >= 0); + REQUIRE(H5Pset_chunk(properties, static_cast(shape.size()), fixture.physicalChunks.data()) >= 0); + REQUIRE(H5Pset_fill_value(properties, H5T_NATIVE_UINT16, &fillValue) >= 0); + if(filter != 0) + { + REQUIRE(H5Pset_deflate(properties, 5) >= 0); + } + dataset = H5Dcreate2(file, "data", H5T_NATIVE_UINT16, space, H5P_DEFAULT, properties, H5P_DEFAULT); + REQUIRE(dataset >= 0); + if(!fixture.sparse) + { + REQUIRE(H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, values.data()) >= 0); + } + REQUIRE(H5Fflush(file, H5F_SCOPE_LOCAL) >= 0); + } + const std::vector tupleChunks(fixture.physicalChunks.begin(), fixture.physicalChunks.begin() + static_cast(fixture.tupleShape.size())); + ParallelChunkCodec codec(path, "data", fixture.tupleShape, tupleChunks, fixture.componentShape, sizeof(uint16), dataset, codecMemoryType()); + constexpr usize guardValues = 16; + constexpr uint16 leadingGuard = 0xD1D1; + constexpr uint16 trailingGuard = 0xE2E2; + constexpr uint16 unfilled = 0xB6B6; + std::vector guardedOutput(guardValues + fixture.expected.size() + guardValues, unfilled); + std::fill_n(guardedOutput.begin(), guardValues, leadingGuard); + std::fill_n(guardedOutput.end() - guardValues, guardValues, trailingGuard); + const auto destination = nonstd::span(reinterpret_cast(guardedOutput.data() + guardValues), fixture.expected.size() * sizeof(uint16)); + BoundedCodecObservation observation; + const auto previous = SetCodecIoObserverForTesting({&observation, &BoundedCodecObservation::Observe, &BoundedCodecObservation::Fault}); + auto restore = MakeScopeGuard([&]() noexcept { (void)SetCodecIoObserverForTesting(previous); }); + constexpr uint64 allowance = 1024 * 1024; + const auto result = codec.readExtentIntoBufferBounded(fixture.extent, destination, allowance); + REQUIRE(result.valid()); + REQUIRE(result.value()); + REQUIRE(result.warnings().empty()); + REQUIRE(std::equal(fixture.expected.begin(), fixture.expected.end(), guardedOutput.begin() + guardValues)); + REQUIRE(std::all_of(guardedOutput.begin(), guardedOutput.begin() + guardValues, [](uint16 value) { return value == leadingGuard; })); + REQUIRE(std::all_of(guardedOutput.end() - guardValues, guardedOutput.end(), [](uint16 value) { return value == trailingGuard; })); + REQUIRE_FALSE(observation.invalid); + REQUIRE(observation.live == 0); + REQUIRE(observation.acquired == observation.released); + REQUIRE(observation.peak <= allowance - BoundedRead::k_DiagnosticBytes); + if(fixture.sparse) + { + REQUIRE(observation.rawReads == 0); + REQUIRE(observation.decoded == 0); + } + else + { + REQUIRE(observation.rawReads > 0); + REQUIRE((observation.decoded > 0) == (filter != 0)); + } + } + } +} diff --git a/test/PipelineFlushTest.cpp b/test/PipelineFlushTest.cpp new file mode 100644 index 0000000000..8de72c59a1 --- /dev/null +++ b/test/PipelineFlushTest.cpp @@ -0,0 +1,1188 @@ +#include "simplnx/Common/Result.hpp" +#include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataGroup.hpp" +#include "simplnx/DataStructure/DataPath.hpp" +#include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" +#include "simplnx/DataStructure/ListStore.hpp" +#include "simplnx/DataStructure/Messaging/DataRemovedMessage.hpp" +#include "simplnx/DataStructure/NeighborList.hpp" +#include "simplnx/DataStructure/Observers/AbstractDataStructureObserver.hpp" +#include "simplnx/Filter/IFilter.hpp" +#include "simplnx/Pipeline/Pipeline.hpp" +#include "simplnx/Pipeline/PipelineFilter.hpp" +#include "simplnx/UnitTest/UnitTestCommon.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; + +namespace +{ +constexpr int32 k_FilterError = -76101; +constexpr int32 k_FlushError = -76102; +constexpr int32 k_FilterWarning = 76103; +constexpr int32 k_FlushWarning = 76104; +const DataPath k_ValuesPath({"Group", "Values"}); +const DataPath k_ListPath({"Group", "Neighbors"}); + +/** + * @enum StoreKind + * @brief Selects the public store adapter under test. + */ +enum class StoreKind +{ + Numeric, ///< Uses a DataArray with resident values. + List ///< Uses a NeighborList with resident lists. +}; + +/** + * @enum SnapshotFailure + * @brief Selects a copy exception while diagnostic memory is available. + */ +enum class SnapshotFailure +{ + None, ///< Permits shallow copies. + Standard, ///< Throws a runtime error. + Allocation, ///< Throws bad_alloc without exhausting memory. + Unknown ///< Throws a nonstandard exception. +}; + +/** + * @struct FlushControl + * @brief Retains the same diagnostic state across shallow snapshots and retries. + */ +struct FlushControl +{ + Result<> result; + usize checkedCalls = 0; + usize legacyCalls = 0; + std::atomic_bool* cancelOnFlush = nullptr; +}; + +/** + * @class NumericCompletionStore + * @brief Separates legacy calls from checked calls at the pipeline boundary. + */ +class NumericCompletionStore : public DataStore +{ +public: + /** + * @brief Creates two literal values and retains the reporting control. + * @param control Owns counters and selected diagnostics. + */ + explicit NumericCompletionStore(std::shared_ptr control) + : DataStore(ShapeType{2}, ShapeType{1}, int32{0}) + , m_Control(std::move(control)) + { + setValue(0, 11); + setValue(1, 13); + } + + ~NumericCompletionStore() override = default; + NumericCompletionStore(const NumericCompletionStore&) = default; + NumericCompletionStore(NumericCompletionStore&&) = default; + NumericCompletionStore& operator=(const NumericCompletionStore&) = delete; + NumericCompletionStore& operator=(NumericCompletionStore&&) = default; + + /** + * @brief Counts a legacy call without returning checked diagnostics. + */ + void flush() const override + { + ++m_Control->legacyCalls; + if(m_Control->cancelOnFlush != nullptr) + { + m_Control->cancelOnFlush->store(true); + } + } + + /** + * @brief Returns diagnostics and can request cancellation after execute. + * @return The configured completion result. + */ + [[nodiscard]] Result<> flushChecked() const override + { + ++m_Control->checkedCalls; + if(m_Control->cancelOnFlush != nullptr) + { + m_Control->cancelOnFlush->store(true); + } + return m_Control->result; + } + +private: + std::shared_ptr m_Control; +}; + +/** + * @class ListCompletionStore + * @brief Returns checked diagnostics through a real public NeighborList. + */ +class ListCompletionStore : public ListStore +{ +public: + /** + * @brief Creates two literal lists and retains the reporting control. + * @param control Owns counters and selected diagnostics. + */ + explicit ListCompletionStore(std::shared_ptr control) + : ListStore(ShapeType{2}) + , m_Control(std::move(control)) + { + setList(0, std::vector{11, 13}); + setList(1, std::vector{17}); + } + + ~ListCompletionStore() override = default; + ListCompletionStore(const ListCompletionStore&) = default; + ListCompletionStore(ListCompletionStore&&) = default; + ListCompletionStore& operator=(const ListCompletionStore&) = delete; + ListCompletionStore& operator=(ListCompletionStore&&) = delete; + + /** + * @brief Returns the selected list completion diagnostics. + * @return The configured completion result. + */ + [[nodiscard]] Result<> flushChecked() const override + { + ++m_Control->checkedCalls; + return m_Control->result; + } + +private: + std::shared_ptr m_Control; +}; + +/** + * @struct FilterControl + * @brief Retains execute results and hooks across filter clones. + */ +struct FilterControl +{ + Result<> result; + usize executeCalls = 0; + bool throwAllocation = false; + bool invalidEmptyPreflight = false; + std::function onExecute; +}; + +/** + * @class CompletionFilter + * @brief Exercises normal IFilter execution without private backend dependencies. + */ +class CompletionFilter : public IFilter +{ +public: + /** + * @brief Retains the selected execute behavior. + * @param control Owns the result, call count, and optional hook. + */ + explicit CompletionFilter(std::shared_ptr control) + : m_Control(std::move(control)) + { + } + + ~CompletionFilter() override = default; + CompletionFilter(const CompletionFilter&) = delete; + CompletionFilter(CompletionFilter&&) = delete; + CompletionFilter& operator=(const CompletionFilter&) = delete; + CompletionFilter& operator=(CompletionFilter&&) = delete; + + std::string name() const override + { + return "CompletionFilter"; + } + std::string className() const override + { + return "CompletionFilter"; + } + Uuid uuid() const override + { + return *Uuid::FromString("82a26d85-21a7-429a-a208-a5f0ba24a6bb"); + } + std::string humanName() const override + { + return "Completion Filter"; + } + VersionType parametersVersion() const override + { + return 1; + } + + /** + * @brief Uses no parameters so validation cannot select another failure phase. + * @return An empty parameter set. + */ + Parameters parameters() const override + { + return {}; + } + + /** + * @brief Shares the same control with the cloned filter. + * @return A new filter with the existing control. + */ + UniquePointer clone() const override + { + return std::make_unique(m_Control); + } + +protected: + /** + * @brief Supplies an output-value sentinel without changing the input structure. + * @param dataStructure Provides existing test objects. + * @param filterArgs Contains no parameters. + * @param messageHandler Is unused by this fixture. + * @param shouldCancel Is checked by the public IFilter wrapper. + * @param executionContext Is supplied by the real parent pipeline. + * @return Successful actions and a literal output value. + */ + PreflightResult preflightImpl([[maybe_unused]] const DataStructure& dataStructure, [[maybe_unused]] const Arguments& filterArgs, [[maybe_unused]] const MessageHandler& messageHandler, + [[maybe_unused]] const std::atomic_bool& shouldCancel, [[maybe_unused]] const ExecutionContext& executionContext) const override + { + PreflightResult result{}; + result.outputValues.push_back({"Completion value", "preserved"}); + if(m_Control->invalidEmptyPreflight) + { + result.outputActions.m_Expected = nonstd::make_unexpected(ErrorCollection{}); + result.outputActions.warnings().push_back({k_FilterWarning, "invalid-empty-original-warning"}); + } + return result; + } + + /** + * @brief Runs the optional phase hook and returns selected execute diagnostics. + * @param dataStructure Keeps the same objects for the completion boundary. + * @param filterArgs Contains no parameters. + * @param pipelineNode Identifies the real parented node. + * @param messageHandler Is unused by this fixture. + * @param shouldCancel Is checked by the public IFilter wrapper. + * @param executionContext Is supplied by the real parent pipeline. + * @return The configured execute result. + * @throws std::bad_alloc If the synthetic execute allocation fault is enabled. + */ + Result<> executeImpl([[maybe_unused]] DataStructure& dataStructure, [[maybe_unused]] const Arguments& filterArgs, [[maybe_unused]] const PipelineFilter* pipelineNode, + [[maybe_unused]] const MessageHandler& messageHandler, [[maybe_unused]] const std::atomic_bool& shouldCancel, + [[maybe_unused]] const ExecutionContext& executionContext) const override + { + ++m_Control->executeCalls; + if(m_Control->onExecute) + { + m_Control->onExecute(); + } + if(m_Control->throwAllocation) + { + throw std::bad_alloc{}; + } + return m_Control->result; + } + +private: + std::shared_ptr m_Control; +}; + +/** + * @class InterfaceCompletionNode + * @brief Distinguishes the additive checked virtual from the legacy virtual. + */ +class InterfaceCompletionNode : public PipelineFilter +{ +public: + using PipelineFilter::PipelineFilter; + + ~InterfaceCompletionNode() override = default; + InterfaceCompletionNode(const InterfaceCompletionNode&) = delete; + InterfaceCompletionNode(InterfaceCompletionNode&&) = delete; + InterfaceCompletionNode& operator=(const InterfaceCompletionNode&) = delete; + InterfaceCompletionNode& operator=(InterfaceCompletionNode&&) = delete; + + bool overrideCompletion = false; + Result<> completionResult; + usize checkedCalls = 0; + usize legacyCalls = 0; + + /** + * @brief Exposes the retained legacy contract for a direct compatibility check. + * @param dataStructure Supplies the original data. + */ + void completeLegacy(DataStructure& dataStructure) + { + endExecution(dataStructure); + } + +protected: + /** + * @brief Counts checked completion and optionally returns an extension result. + * @param dataStructure Supplies the actual pipeline data. + * @return Configured extension diagnostics or the base completion result. + */ + Result<> endExecutionChecked(DataStructure& dataStructure) override + { + ++checkedCalls; + return overrideCompletion ? completionResult : AbstractPipelineNode::endExecutionChecked(dataStructure); + } + + /** + * @brief Counts the legacy virtual while retaining its original flush contract. + * @param dataStructure Supplies data to flush and snapshot. + */ + void endExecution(DataStructure& dataStructure) override + { + ++legacyCalls; + AbstractPipelineNode::endExecution(dataStructure); + } +}; + +/** + * @class RemovalWitness + * @brief Records existing temporary-copy notifications without throwing or changing the source. + */ +class RemovalWitness : public AbstractDataStructureObserver +{ +public: + RemovalWitness() = default; + ~RemovalWitness() override = default; + RemovalWitness(const RemovalWitness&) = delete; + RemovalWitness(RemovalWitness&&) = delete; + RemovalWitness& operator=(const RemovalWitness&) = delete; + RemovalWitness& operator=(RemovalWitness&&) = delete; + + usize liveRemovalCount = 0; + DataObject::IdType lastLiveId = 0; + + /** + * @brief Counts removal messages whose original object remains present. + * @param target Supplies the source ID lookup. + * @param message Identifies the destroyed temporary clone. + */ + void onNotify(DataStructure* target, const std::shared_ptr& message) noexcept override + { + const auto* removed = dynamic_cast(message.get()); + if(removed != nullptr && target->getData(removed->getId()) != nullptr) + { + ++liveRemovalCount; + lastLiveId = removed->getId(); + } + } +}; + +/** + * @struct SnapshotControl + * @brief Arms copy faults without throwing from destructors or observers. + */ +struct SnapshotControl +{ + SnapshotFailure failure = SnapshotFailure::None; + usize copyCalls = 0; +}; + +/** + * @class SnapshotGroup + * @brief Preserves a shared copy-fault control in each successful shallow copy. + */ +class SnapshotGroup : public DataGroup +{ +public: + /** + * @brief Creates an insertable group after earlier normal test objects. + * @param dataStructure Owns the group after insertion. + * @param control Selects the copy failure. + */ + SnapshotGroup(DataStructure& dataStructure, std::shared_ptr control) + : DataGroup(dataStructure, "CopyFault") + , m_Control(std::move(control)) + { + } + + /** + * @brief Copies the group topology and shares only the fault control. + * @param other Supplies identity and the shared control. + */ + SnapshotGroup(const SnapshotGroup& other) + : DataGroup(other) + , m_Control(other.m_Control) + { + } + + ~SnapshotGroup() override = default; + + // Rvalues retain the source topology and shared fault control through the copy contract. + SnapshotGroup(SnapshotGroup&& other) + : SnapshotGroup(static_cast(other)) + { + } + + SnapshotGroup& operator=(const SnapshotGroup&) = delete; + SnapshotGroup& operator=(SnapshotGroup&&) = delete; + + /** + * @brief Copies the object or throws the selected synthetic exception. + * @return A caller-owned shallow copy. + * @throws std::runtime_error If an ordinary copy fault is armed. + * @throws std::bad_alloc If an allocation copy fault is armed. + * @throws int If a nonstandard copy fault is armed. + */ + DataObject* shallowCopy() override + { + ++m_Control->copyCalls; + switch(m_Control->failure) + { + case SnapshotFailure::Standard: + throw std::runtime_error("copy-sentinel: shallow snapshot failed"); + case SnapshotFailure::Allocation: + throw std::bad_alloc{}; + case SnapshotFailure::Unknown: + throw 41; + case SnapshotFailure::None: + return new SnapshotGroup(*this); + } + throw std::logic_error("Unknown snapshot test mode"); + } + +private: + std::shared_ptr m_Control; +}; + +/** + * @struct DetailRecord + * @brief Captures the emitted collection order independently of GUI sorting. + */ +struct DetailRecord +{ + int32 index = 0; + WarningCollection warnings; + ErrorCollection errors; +}; + +/** + * @struct CompletionTrace + * @brief Owns signal connections only while the referenced pipeline is alive. + */ +struct CompletionTrace +{ + /** + * @brief Observes a root pipeline and an optional first filter. + * @param pipeline Emits root completion and cancellation events. + * @param firstNode Emits leaf details, state, and updates when nonnull. + */ + CompletionTrace(Pipeline& pipeline, PipelineFilter* firstNode) + { + connections.push_back(pipeline.getPipelineRunStateSignal().connect([this](AbstractPipelineNode*, RunState state) { pipelineStates.push_back(state); })); + connections.push_back(pipeline.getCancelledSignal().connect([this]() { ++cancelledCalls; })); + connections.push_back(pipeline.getFilterFaultDetailSignal().connect( + [this](AbstractPipelineNode*, int32 index, const WarningCollection& warnings, const ErrorCollection& errors) { rootDetails.push_back({index, warnings, errors}); })); + if(firstNode != nullptr) + { + connections.push_back(firstNode->getFilterRunStateSignal().connect([this](AbstractPipelineNode*, int32, RunState state) { nodeStates.push_back(state); })); + connections.push_back(firstNode->getFilterUpdateSignal().connect([this](AbstractPipelineNode*, int32, const std::string& message) { updates.push_back(message); })); + connections.push_back(firstNode->getFilterFaultDetailSignal().connect( + [this](AbstractPipelineNode*, int32 index, const WarningCollection& warnings, const ErrorCollection& errors) { nodeDetails.push_back({index, warnings, errors}); })); + } + } + + ~CompletionTrace() = default; + CompletionTrace(const CompletionTrace&) = default; + CompletionTrace(CompletionTrace&&) = default; + CompletionTrace& operator=(const CompletionTrace&) = default; + CompletionTrace& operator=(CompletionTrace&&) = default; + + std::vector pipelineStates; + std::vector nodeStates; + std::vector updates; + std::vector nodeDetails; + std::vector rootDetails; + usize cancelledCalls = 0; + std::vector connections; +}; + +/** + * @struct CompletionFixture + * @brief Keeps source objects and their reporting stores alive through retries. + */ +struct CompletionFixture +{ + /** + * @brief Creates literal input and properly parented real pipeline nodes. + * @param kind Selects the reporting numeric or list adapter. + * @param includeSecond Adds the later-filter execution witness when true. + */ + explicit CompletionFixture(StoreKind kind, bool includeSecond = true) + : storeKind(kind) + { + UnitTest::LoadPlugins(); + const auto* group = DataGroup::Create(dataStructure, "Group"); + REQUIRE(group != nullptr); + if(kind == StoreKind::Numeric) + { + numericStore = std::make_shared(flushControl); + REQUIRE(DataArray::Create(dataStructure, "Values", numericStore, group->getId()) != nullptr); + } + else + { + listStore = std::make_shared(flushControl); + REQUIRE(NeighborList::Create(dataStructure, "Neighbors", listStore, group->getId()) != nullptr); + } + REQUIRE(pipeline.push_back(std::make_unique(firstControl), Arguments{})); + firstNode = dynamic_cast(pipeline.at(0)); + REQUIRE(firstNode != nullptr); + firstNode->setIndex(0); + if(includeSecond) + { + REQUIRE(pipeline.push_back(std::make_unique(secondControl), Arguments{})); + auto* secondNode = dynamic_cast(pipeline.at(1)); + REQUIRE(secondNode != nullptr); + secondNode->setIndex(1); + } + } + + ~CompletionFixture() = default; + CompletionFixture(const CompletionFixture&) = default; + CompletionFixture(CompletionFixture&&) = default; + CompletionFixture& operator=(const CompletionFixture&) = default; + CompletionFixture& operator=(CompletionFixture&&) = default; + + StoreKind storeKind; + DataStructure dataStructure; + Pipeline pipeline; + std::shared_ptr flushControl = std::make_shared(); + std::shared_ptr firstControl = std::make_shared(); + std::shared_ptr secondControl = std::make_shared(); + std::shared_ptr numericStore; + std::shared_ptr listStore; + PipelineFilter* firstNode = nullptr; +}; + +/** + * @brief Checks literal contents and shared storage in a completed shallow snapshot. + * @param fixture Owns the source structure and original stores. + * @param snapshot Supplies the captured object topology. + */ +void CheckSnapshot(const CompletionFixture& fixture, const DataStructure& snapshot) +{ + const DataPath& path = fixture.storeKind == StoreKind::Numeric ? k_ValuesPath : k_ListPath; + const auto* sourceObject = fixture.dataStructure.getData(path); + const auto* copiedObject = snapshot.getData(path); + REQUIRE(sourceObject != nullptr); + REQUIRE(copiedObject != nullptr); + CHECK(copiedObject != sourceObject); + CHECK(copiedObject->getId() == sourceObject->getId()); + CHECK(copiedObject->getParentIds() == sourceObject->getParentIds()); + CHECK(copiedObject->getDataStructure() == &snapshot); + CHECK(sourceObject->getDataStructure() == &fixture.dataStructure); + CHECK(snapshot.getNextId() == fixture.dataStructure.getNextId()); + CHECK(&snapshot.formatResolver() == &fixture.dataStructure.formatResolver()); + if(fixture.storeKind == StoreKind::Numeric) + { + const auto* array = snapshot.getDataAs>(path); + REQUIRE(array != nullptr); + CHECK(&array->getDataStoreRef() == fixture.numericStore.get()); + CHECK(array->getDataStoreRef().getValue(0) == 11); + CHECK(array->getDataStoreRef().getValue(1) == 13); + } + else + { + const auto* lists = snapshot.getDataAs>(path); + REQUIRE(lists != nullptr); + CHECK(lists->getIListStore() == fixture.listStore.get()); + CHECK(lists->getList(0) == std::vector{11, 13}); + CHECK(lists->getList(1) == std::vector{17}); + } +} + +/** + * @brief Checks normal terminal signals and the retained output-value sentinel. + * @param trace Contains events from the completed or failed execution. + * @param firstNode Owns the execute output values. + */ +void CheckNodeCompletion(const CompletionTrace& trace, const PipelineFilter& firstNode) +{ + REQUIRE_FALSE(trace.nodeStates.empty()); + CHECK(trace.nodeStates.back() == RunState::Idle); + REQUIRE_FALSE(trace.pipelineStates.empty()); + CHECK(trace.pipelineStates.back() == RunState::Idle); + REQUIRE_FALSE(trace.updates.empty()); + CHECK(trace.updates.back() == "End"); + const auto& values = firstNode.getPreflightValues(); + REQUIRE(values.size() == 1); + CHECK(values[0].name == "Completion value"); + CHECK(values[0].value == "preserved"); +} + +/** + * @brief Installs the copy-fault group after the normal literal objects. + * @param fixture Owns the source structure. + * @return Shared control for the inserted group and its shallow copies. + */ +std::shared_ptr AddSnapshotGroup(CompletionFixture& fixture) +{ + auto control = std::make_shared(); + auto group = std::make_shared(fixture.dataStructure, control); + REQUIRE(fixture.dataStructure.insert(group, DataPath{})); + return control; +} +} // namespace + +TEST_CASE("PipelineFlush retains execute diagnostics and fails before later filters", "[simplnx][PipelineFlush]") +{ + const auto kind = GENERATE(StoreKind::Numeric, StoreKind::List); + const bool originalError = GENERATE(false, true); + DYNAMIC_SECTION("Store " << static_cast(kind) << " original error " << originalError) + { + CompletionFixture fixture(kind); + fixture.firstControl->result.warnings().push_back({k_FilterWarning, "execute-warning-sentinel"}); + if(originalError) + { + fixture.firstControl->result.m_Expected = nonstd::make_unexpected(ErrorCollection{{k_FilterError, "execute-error-sentinel"}}); + } + fixture.flushControl->result = MakeErrorResult(k_FlushError, "backing.h5:/values: terminal-flush-sentinel"); + fixture.flushControl->result.warnings().push_back({k_FlushWarning, "flush-warning-sentinel"}); + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = true; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK_FALSE(succeeded); + CHECK(fixture.firstControl->executeCalls == 1); + CHECK(fixture.secondControl->executeCalls == 0); + CHECK(fixture.flushControl->checkedCalls == 1); + CHECK(fixture.flushControl->legacyCalls == 0); + CHECK(fixture.firstNode->hasErrors()); + CHECK_FALSE(fixture.firstNode->hasWarnings()); + CHECK(fixture.pipeline.hasErrors()); + CHECK_FALSE(fixture.pipeline.hasWarnings()); + + const auto errors = fixture.firstNode->getErrors(); + REQUIRE(errors.size() == (originalError ? 2 : 1)); + if(originalError) + { + CHECK(errors.front().code == k_FilterError); + CHECK(errors.front().message == "execute-error-sentinel"); + } + CHECK(errors.back().code == k_FlushError); + CHECK(errors.back().message.find("backing.h5:/values") != std::string::npos); + CHECK(errors.back().message.find(kind == StoreKind::Numeric ? "Values" : "Neighbors") != std::string::npos); + const auto warnings = fixture.firstNode->getWarnings(); + REQUIRE(warnings.size() == 2); + CHECK(warnings[0].code == k_FilterWarning); + CHECK(warnings[1].code == k_FlushWarning); + REQUIRE(trace.nodeDetails.size() == 1); + CHECK(trace.nodeDetails[0].index == 0); + REQUIRE(trace.nodeDetails[0].errors.size() == errors.size()); + for(usize index = 0; index < errors.size(); ++index) + { + CHECK(trace.nodeDetails[0].errors[index].code == errors[index].code); + CHECK(trace.nodeDetails[0].errors[index].message == errors[index].message); + } + CHECK(trace.rootDetails.empty()); + CheckNodeCompletion(trace, *fixture.firstNode); + CheckSnapshot(fixture, fixture.firstNode->getDataStructure()); + CheckSnapshot(fixture, fixture.pipeline.getDataStructure()); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush retries the same stores and clears completion faults", "[simplnx][PipelineFlush]") +{ + const auto kind = GENERATE(StoreKind::Numeric, StoreKind::List); + DYNAMIC_SECTION("Store " << static_cast(kind)) + { + CompletionFixture fixture(kind); + fixture.flushControl->result = MakeErrorResult(k_FlushError, "retry-flush-sentinel"); + bool succeeded = true; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK_FALSE(succeeded); + CHECK(fixture.secondControl->executeCalls == 0); + fixture.flushControl->result = {}; + CompletionTrace retryTrace(fixture.pipeline, fixture.firstNode); + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK(succeeded); + CHECK(fixture.firstControl->executeCalls == 2); + CHECK(fixture.secondControl->executeCalls == 1); + CHECK(fixture.flushControl->checkedCalls == 3); + CHECK(fixture.flushControl->legacyCalls == 0); + CHECK_FALSE(fixture.firstNode->hasErrors()); + CHECK_FALSE(fixture.pipeline.hasErrors()); + CHECK(fixture.firstNode->getErrors().empty()); + CHECK(retryTrace.nodeDetails.empty()); + CHECK(retryTrace.rootDetails.empty()); + CheckNodeCompletion(retryTrace, *fixture.firstNode); + CheckSnapshot(fixture, fixture.firstNode->getDataStructure()); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush warning survives a later clean completion", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric); + fixture.flushControl->result = MakeWarningVoidResult(k_FlushWarning, "early-warning-sentinel"); + fixture.secondControl->onExecute = [&fixture]() { fixture.flushControl->result = {}; }; + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = false; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK(succeeded); + CHECK(fixture.secondControl->executeCalls == 1); + CHECK(fixture.firstNode->hasWarnings()); + CHECK(fixture.pipeline.hasWarnings()); + CHECK_FALSE(fixture.pipeline.hasErrors()); + REQUIRE(trace.nodeDetails.size() == 1); + REQUIRE(trace.nodeDetails[0].warnings.size() == 1); + CHECK(trace.nodeDetails[0].warnings[0].code == k_FlushWarning); + CHECK(trace.nodeDetails[0].errors.empty()); + CHECK(trace.rootDetails.empty()); + CheckNodeCompletion(trace, *fixture.firstNode); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +TEST_CASE("PipelineFlush preserves allocation classification and earlier errors", "[simplnx][PipelineFlush]") +{ + const bool allocationInExecute = GENERATE(false, true); + DYNAMIC_SECTION("Allocation in execute " << allocationInExecute) + { + CompletionFixture fixture(StoreKind::Numeric); + fixture.firstControl->throwAllocation = allocationInExecute; + if(!allocationInExecute) + { + fixture.firstControl->result = MakeErrorResult(k_FilterError, "original-before-allocation"); + } + fixture.flushControl->result = MakeErrorResult(allocationInExecute ? k_FlushError : -272, "completion-allocation-routing"); + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = true; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK_FALSE(succeeded); + CHECK(fixture.secondControl->executeCalls == 0); + const auto errors = fixture.firstNode->getErrors(); + REQUIRE(errors.size() == 2); + CHECK(errors[0].code == (allocationInExecute ? -272 : k_FilterError)); + CHECK(errors[1].code == (allocationInExecute ? k_FlushError : -272)); + CheckNodeCompletion(trace, *fixture.firstNode); + CheckSnapshot(fixture, fixture.firstNode->getDataStructure()); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush cancellation at the flush boundary retains completion failure", "[simplnx][PipelineFlush]") +{ + const bool failFlush = GENERATE(false, true); + DYNAMIC_SECTION("Flush fails " << failFlush) + { + CompletionFixture fixture(StoreKind::Numeric); + std::atomic_bool cancelled = false; + fixture.flushControl->cancelOnFlush = &cancelled; + if(failFlush) + { + fixture.flushControl->result = MakeErrorResult(k_FlushError, "flush-after-cancel-sentinel"); + } + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = !failFlush; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, cancelled)); + CHECK(succeeded == !failFlush); + CHECK(cancelled.load()); + CHECK(trace.cancelledCalls == 1); + CHECK(fixture.secondControl->executeCalls == 0); + CHECK(fixture.pipeline.hasErrors() == failFlush); + if(failFlush) + { + const auto errors = fixture.firstNode->getErrors(); + REQUIRE(errors.size() == 1); + CHECK(errors[0].code == k_FlushError); + } + CheckNodeCompletion(trace, *fixture.firstNode); + fixture.flushControl->cancelOnFlush = nullptr; + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush leaves execute-stage cancellation behavior unchanged", "[simplnx][PipelineFlush]") +{ + const bool originalError = GENERATE(false, true); + DYNAMIC_SECTION("Original execute error " << originalError) + { + CompletionFixture fixture(StoreKind::Numeric); + std::atomic_bool cancelled = false; + fixture.firstControl->onExecute = [&cancelled]() { cancelled.store(true); }; + if(originalError) + { + fixture.firstControl->result = MakeErrorResult(k_FilterError, "execute-cancel-error-sentinel"); + } + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = false; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, cancelled)); + CHECK(succeeded); + CHECK_FALSE(fixture.pipeline.hasErrors()); + CHECK_FALSE(fixture.pipeline.hasCompletionErrors()); + CHECK_FALSE(fixture.firstNode->hasCompletionErrors()); + CHECK(trace.cancelledCalls == 1); + CHECK(fixture.secondControl->executeCalls == 0); + const auto errors = fixture.firstNode->getErrors(); + REQUIRE(errors.size() == 1); + CHECK(errors[0].code == (originalError ? k_FilterError : -1)); + CheckNodeCompletion(trace, *fixture.firstNode); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush repeated snapshot exceptions retain diagnostics and settle", "[simplnx][PipelineFlush]") +{ + const auto failure = GENERATE(SnapshotFailure::Standard, SnapshotFailure::Allocation, SnapshotFailure::Unknown); + DYNAMIC_SECTION("Copy failure " << static_cast(failure)) + { + CompletionFixture fixture(StoreKind::Numeric); + auto snapshotControl = AddSnapshotGroup(fixture); + fixture.firstControl->result = MakeErrorResult(k_FilterError, "before-copy-error-sentinel"); + fixture.flushControl->result = MakeErrorResult(k_FlushError, "before-copy-flush-sentinel"); + fixture.firstControl->onExecute = [snapshotControl, failure]() { snapshotControl->failure = failure; }; + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = true; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + snapshotControl->failure = SnapshotFailure::None; + CHECK_FALSE(succeeded); + CHECK(fixture.secondControl->executeCalls == 0); + CHECK(snapshotControl->copyCalls >= 2); + CHECK(fixture.pipeline.hasErrors()); + const auto errors = fixture.firstNode->getErrors(); + REQUIRE(errors.size() == 3); + CHECK(errors[0].code == k_FilterError); + CHECK(errors[1].code == k_FlushError); + CHECK(errors[2].code == (failure == SnapshotFailure::Allocation ? -272 : -6071)); + REQUIRE(trace.rootDetails.size() == 1); + CHECK(trace.rootDetails[0].index == -1); + REQUIRE(trace.rootDetails[0].errors.size() == 1); + CHECK(trace.rootDetails[0].errors[0].code == (failure == SnapshotFailure::Allocation ? -272 : -6071)); + CHECK(trace.rootDetails[0].warnings.empty()); + CheckNodeCompletion(trace, *fixture.firstNode); + const auto* sourceGroup = fixture.dataStructure.getData(DataPath({"CopyFault"})); + REQUIRE(sourceGroup != nullptr); + CHECK(sourceGroup->getDataStructure() == &fixture.dataStructure); + CHECK(fixture.numericStore->getValue(0) == 11); + CHECK(fixture.numericStore->getValue(1) == 13); + CHECK(fixture.firstNode->getDataStructure().getData(DataPath({"CopyFault"})) == nullptr); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush root snapshot errors use a pipeline event and preserve prior snapshot", "[simplnx][PipelineFlush]") +{ + const auto failure = GENERATE(SnapshotFailure::Standard, SnapshotFailure::Allocation); + DYNAMIC_SECTION("Root copy failure " << static_cast(failure)) + { + CompletionFixture fixture(StoreKind::Numeric, false); + auto snapshotControl = AddSnapshotGroup(fixture); + bool succeeded = false; + REQUIRE_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + REQUIRE(succeeded); + const auto oldRootNextId = fixture.pipeline.getDataStructure().getNextId(); + REQUIRE(DataGroup::Create(fixture.dataStructure, "NewGroup") != nullptr); + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + // End follows the successful node snapshot, so only the root snapshot sees this fault. + nod::scoped_connection armRoot = fixture.firstNode->getFilterUpdateSignal().connect([snapshotControl, failure](AbstractPipelineNode*, int32, const std::string& message) { + if(message == "End") + { + snapshotControl->failure = failure; + } + }); + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + snapshotControl->failure = SnapshotFailure::None; + CHECK_FALSE(succeeded); + CHECK(fixture.pipeline.hasErrors()); + CHECK_FALSE(fixture.firstNode->hasErrors()); + CHECK(trace.nodeDetails.empty()); + REQUIRE(trace.rootDetails.size() == 1); + CHECK(trace.rootDetails[0].index == -1); + REQUIRE(trace.rootDetails[0].errors.size() == 1); + CHECK(trace.rootDetails[0].errors[0].code == (failure == SnapshotFailure::Allocation ? -272 : -6071)); + CHECK(fixture.pipeline.getDataStructure().getData(DataPath({"NewGroup"})) == nullptr); + CHECK(fixture.pipeline.getDataStructure().getNextId() == oldRootNextId); + REQUIRE(fixture.firstNode->getDataStructure().getData(DataPath({"NewGroup"})) != nullptr); + CheckSnapshot(fixture, fixture.firstNode->getDataStructure()); + CheckNodeCompletion(trace, *fixture.firstNode); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); + } +} + +TEST_CASE("PipelineFlush empty pipeline reports a root snapshot exception", "[simplnx][PipelineFlush]") +{ + UnitTest::LoadPlugins(); + DataStructure dataStructure; + auto control = std::make_shared(); + auto group = std::make_shared(dataStructure, control); + REQUIRE(dataStructure.insert(group, DataPath{})); + control->failure = SnapshotFailure::Standard; + Pipeline pipeline; + CompletionTrace trace(pipeline, nullptr); + bool succeeded = true; + CHECK_NOTHROW(succeeded = pipeline.execute(dataStructure, false)); + control->failure = SnapshotFailure::None; + CHECK_FALSE(succeeded); + CHECK(pipeline.hasErrors()); + REQUIRE(trace.rootDetails.size() == 1); + CHECK(trace.rootDetails[0].index == -1); + REQUIRE(trace.rootDetails[0].errors.size() == 1); + CHECK(trace.rootDetails[0].errors[0].code == -6071); + REQUIRE_FALSE(trace.pipelineStates.empty()); + CHECK(trace.pipelineStates.back() == RunState::Idle); + CHECK(dataStructure.getData(group->getId()) == group.get()); + CHECK(group->getDataStructure() == &dataStructure); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +TEST_CASE("PipelineFlush snapshot preserves multiply parented object identity", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric, false); + auto* secondParent = DataGroup::Create(fixture.dataStructure, "Second"); + REQUIRE(secondParent != nullptr); + auto* sourceArray = fixture.dataStructure.getDataAs>(k_ValuesPath); + REQUIRE(sourceArray != nullptr); + REQUIRE(fixture.dataStructure.setAdditionalParent(sourceArray->getId(), secondParent->getId())); + const auto sourceId = sourceArray->getId(); + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = false; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK(succeeded); + const auto& snapshot = fixture.firstNode->getDataStructure(); + const auto* firstCopy = snapshot.getData(k_ValuesPath); + const auto* secondCopy = snapshot.getData(DataPath({"Second", "Values"})); + REQUIRE(firstCopy != nullptr); + CHECK(secondCopy == firstCopy); + CHECK(firstCopy->getId() == sourceId); + CHECK(fixture.flushControl->checkedCalls == 1); + CHECK(fixture.dataStructure.getData(sourceId) == sourceArray); + CheckSnapshot(fixture, snapshot); + CheckNodeCompletion(trace, *fixture.firstNode); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +TEST_CASE("PipelineFlush nested cancellation preserves completion failure and clears it on retry", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric, false); + auto nestedPipeline = std::make_shared(std::move(fixture.pipeline)); + Pipeline outerPipeline; + REQUIRE(outerPipeline.push_back(nestedPipeline)); + REQUIRE(outerPipeline.push_back(std::make_unique(fixture.secondControl), Arguments{})); + std::atomic_bool cancelled = false; + fixture.flushControl->cancelOnFlush = &cancelled; + fixture.flushControl->result = MakeErrorResult(k_FlushError, "nested-completion-sentinel"); + CompletionTrace failureTrace(outerPipeline, fixture.firstNode); + bool succeeded = true; + CHECK_NOTHROW(succeeded = outerPipeline.execute(fixture.dataStructure, cancelled)); + CHECK_FALSE(succeeded); + CHECK(outerPipeline.hasErrors()); + CHECK(nestedPipeline->hasErrors()); + CHECK(outerPipeline.hasCompletionErrors()); + CHECK(nestedPipeline->hasCompletionErrors()); + CHECK(fixture.firstNode->hasCompletionErrors()); + CHECK(fixture.secondControl->executeCalls == 0); + CheckNodeCompletion(failureTrace, *fixture.firstNode); + + fixture.flushControl->result = {}; + fixture.flushControl->cancelOnFlush = nullptr; + cancelled.store(false); + CompletionTrace retryTrace(outerPipeline, fixture.firstNode); + CHECK_NOTHROW(succeeded = outerPipeline.execute(fixture.dataStructure, cancelled)); + CHECK(succeeded); + CHECK_FALSE(outerPipeline.hasErrors()); + CHECK_FALSE(nestedPipeline->hasErrors()); + CHECK_FALSE(outerPipeline.hasCompletionErrors()); + CHECK_FALSE(nestedPipeline->hasCompletionErrors()); + CHECK_FALSE(fixture.firstNode->hasCompletionErrors()); + CHECK(fixture.secondControl->executeCalls == 1); + CHECK(fixture.flushControl->checkedCalls == 3); + CheckNodeCompletion(retryTrace, *fixture.firstNode); + CheckSnapshot(fixture, nestedPipeline->getDataStructure()); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +TEST_CASE("PipelineFlush failed node snapshot retains its prior complete topology", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric, false); + auto snapshotControl = AddSnapshotGroup(fixture); + bool succeeded = false; + REQUIRE_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + REQUIRE(succeeded); + const auto oldNodeNextId = fixture.firstNode->getDataStructure().getNextId(); + REQUIRE(DataGroup::Create(fixture.dataStructure, "NewGroup") != nullptr); + fixture.firstControl->onExecute = [snapshotControl]() { snapshotControl->failure = SnapshotFailure::Standard; }; + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + snapshotControl->failure = SnapshotFailure::None; + CHECK_FALSE(succeeded); + CHECK(fixture.firstNode->hasErrors()); + CHECK(fixture.pipeline.hasErrors()); + const auto& retained = fixture.firstNode->getDataStructure(); + CHECK(retained.getData(DataPath({"NewGroup"})) == nullptr); + CHECK(retained.getNextId() == oldNodeNextId); + const auto* retainedArray = retained.getDataAs>(k_ValuesPath); + REQUIRE(retainedArray != nullptr); + CHECK(retainedArray->getDataStructure() == &retained); + CHECK(&retainedArray->getDataStoreRef() == fixture.numericStore.get()); + const auto* sourceArray = fixture.dataStructure.getDataAs>(k_ValuesPath); + REQUIRE(sourceArray != nullptr); + CHECK(sourceArray->getDataStructure() == &fixture.dataStructure); + CHECK(sourceArray->getDataStoreRef().getValue(0) == 11); + CHECK(sourceArray->getDataStoreRef().getValue(1) == 13); + REQUIRE(fixture.dataStructure.getData(DataPath({"NewGroup"})) != nullptr); + CheckNodeCompletion(trace, *fixture.firstNode); + UnitTest::CheckArraysInheritTupleDims(fixture.dataStructure); +} + +TEST_CASE("PipelineFlush checked virtual preserves invalid-empty extension results and legacy completion", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric, false); + Pipeline pipeline; + auto node = std::make_shared(std::make_unique(fixture.firstControl), Arguments{}); + node->setIndex(0); + REQUIRE(pipeline.push_back(node)); + node->overrideCompletion = true; + node->completionResult.m_Expected = nonstd::make_unexpected(ErrorCollection{}); + node->completionResult.warnings().push_back({k_FlushWarning, "empty-extension-warning"}); + CompletionTrace trace(pipeline, node.get()); + CHECK_FALSE(node->hasCompletionErrors()); + CHECK_FALSE(pipeline.execute(fixture.dataStructure, false)); + CHECK(node->hasCompletionErrors()); + CHECK(pipeline.hasCompletionErrors()); + CHECK(pipeline.hasErrors()); + REQUIRE(node->getErrors().size() == 1); + CHECK(node->getErrors()[0].code == -6071); + CHECK(node->getErrors()[0].message.find("checked completion") != std::string::npos); + REQUIRE(node->getWarnings().size() == 1); + CHECK(node->getWarnings()[0].message == "empty-extension-warning"); + CHECK(node->checkedCalls == 1); + CHECK(node->legacyCalls == 0); + CheckNodeCompletion(trace, *node); + + node->overrideCompletion = false; + CHECK(pipeline.execute(fixture.dataStructure, false)); + CHECK_FALSE(node->hasCompletionErrors()); + CHECK_FALSE(pipeline.hasCompletionErrors()); + CHECK(node->getErrors().empty()); + CHECK(node->checkedCalls == 2); + CHECK(node->legacyCalls == 0); + CHECK(fixture.flushControl->checkedCalls == 1); + node->completeLegacy(fixture.dataStructure); + CHECK(node->legacyCalls == 1); + CHECK(fixture.flushControl->legacyCalls == 1); + CHECK(fixture.flushControl->checkedCalls == 1); +} + +TEST_CASE("PipelineFlush preserves original invalid-empty filter result validity", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric); + // The IFilter wrapper returns an invalid preflight result without an intervening diagnostic merge. + fixture.firstControl->invalidEmptyPreflight = true; + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + CHECK_FALSE(fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK(fixture.pipeline.hasErrors()); + CHECK(fixture.firstNode->hasErrors()); + CHECK_FALSE(fixture.firstNode->hasCompletionErrors()); + CHECK_FALSE(fixture.pipeline.hasCompletionErrors()); + CHECK(fixture.firstNode->getErrors().empty()); + REQUIRE(fixture.firstNode->getWarnings().size() == 1); + CHECK(fixture.firstNode->getWarnings()[0].message == "invalid-empty-original-warning"); + CHECK(fixture.firstControl->executeCalls == 0); + CHECK(fixture.secondControl->executeCalls == 0); + CHECK(fixture.flushControl->checkedCalls == 1); + CheckNodeCompletion(trace, *fixture.firstNode); +} + +TEST_CASE("PipelineFlush completion outcome follows actual execution lifecycle", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric); + CHECK_FALSE(fixture.pipeline.hasCompletionErrors()); + CHECK_FALSE(fixture.firstNode->hasCompletionErrors()); + fixture.flushControl->result = MakeErrorResult(k_FlushError, "lifecycle-failure"); + REQUIRE_FALSE(fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK(fixture.pipeline.hasCompletionErrors()); + CHECK(fixture.firstNode->hasCompletionErrors()); + CHECK_FALSE(fixture.pipeline.executeFrom(fixture.pipeline.size() + 1, fixture.dataStructure, false)); + CHECK(fixture.pipeline.hasCompletionErrors()); + + auto copy = fixture.pipeline.deepCopy(); + CHECK_FALSE(copy->hasCompletionErrors()); + + fixture.flushControl->result = {}; + REQUIRE(fixture.pipeline.preflight(fixture.dataStructure, false)); + CHECK(fixture.pipeline.hasCompletionErrors()); + CHECK(fixture.firstNode->hasCompletionErrors()); + REQUIRE(fixture.pipeline.executeFrom(1, fixture.dataStructure, false)); + CHECK_FALSE(fixture.pipeline.hasCompletionErrors()); + CHECK(fixture.firstNode->hasCompletionErrors()); + fixture.firstNode->setDisabled(true); + REQUIRE(fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK_FALSE(fixture.pipeline.hasCompletionErrors()); + CHECK(fixture.firstNode->hasCompletionErrors()); + fixture.firstNode->setDisabled(false); + REQUIRE(fixture.pipeline.execute(fixture.dataStructure, false)); + CHECK_FALSE(fixture.firstNode->hasCompletionErrors()); + CHECK_FALSE(fixture.pipeline.hasCompletionErrors()); + + fixture.flushControl->result = MakeErrorResult(k_FlushError, "copy-lifecycle-failure"); + REQUIRE_FALSE(fixture.pipeline.execute(fixture.dataStructure, false)); + Pipeline copied(fixture.pipeline); + CHECK_FALSE(copied.hasCompletionErrors()); + Pipeline moved(std::move(copied)); + CHECK_FALSE(moved.hasCompletionErrors()); +} + +TEST_CASE("PipelineFlush failed snapshot notification leaves live source topology usable", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric, false); + auto control = AddSnapshotGroup(fixture); + REQUIRE(fixture.pipeline.execute(fixture.dataStructure, false)); + const auto* sourceArray = fixture.dataStructure.getData(k_ValuesPath); + REQUIRE(sourceArray != nullptr); + const auto sourceId = sourceArray->getId(); + const auto parentIds = sourceArray->getParentIds(); + RemovalWitness witness; + witness.startObservingStructure(&fixture.dataStructure); + fixture.firstControl->onExecute = [control]() { control->failure = SnapshotFailure::Standard; }; + CompletionTrace trace(fixture.pipeline, fixture.firstNode); + bool succeeded = true; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + control->failure = SnapshotFailure::None; + CHECK_FALSE(succeeded); + // Temporary clones can emit source removals before copy construction rebinds their owner. + CHECK(witness.liveRemovalCount > 0); + CHECK(fixture.dataStructure.getData(witness.lastLiveId) != nullptr); + CHECK(fixture.dataStructure.getData(sourceId) == sourceArray); + CHECK(fixture.dataStructure.getData(k_ValuesPath) == sourceArray); + CHECK(sourceArray->getParentIds() == parentIds); + CHECK(sourceArray->getDataStructure() == &fixture.dataStructure); + CHECK(fixture.numericStore->getValue(0) == 11); + CHECK(fixture.numericStore->getValue(1) == 13); + CheckSnapshot(fixture, fixture.firstNode->getDataStructure()); + CheckNodeCompletion(trace, *fixture.firstNode); + witness.stopObservingStructure(); +} + +TEST_CASE("PipelineFlush disabled pipeline still reports final snapshot errors", "[simplnx][PipelineFlush]") +{ + CompletionFixture fixture(StoreKind::Numeric); + auto control = AddSnapshotGroup(fixture); + for(const auto& node : fixture.pipeline) + { + node->setDisabled(true); + } + control->failure = SnapshotFailure::Unknown; + CompletionTrace trace(fixture.pipeline, nullptr); + bool succeeded = true; + CHECK_NOTHROW(succeeded = fixture.pipeline.execute(fixture.dataStructure, false)); + control->failure = SnapshotFailure::None; + CHECK_FALSE(succeeded); + CHECK(fixture.pipeline.hasCompletionErrors()); + CHECK(fixture.firstControl->executeCalls == 0); + CHECK(fixture.secondControl->executeCalls == 0); + CHECK(fixture.flushControl->checkedCalls == 0); + REQUIRE(trace.rootDetails.size() == 1); + CHECK(trace.rootDetails[0].index == -1); + REQUIRE(trace.rootDetails[0].errors.size() == 1); + CHECK(trace.rootDetails[0].errors[0].code == -6071); + REQUIRE_FALSE(trace.pipelineStates.empty()); + CHECK(trace.pipelineStates.back() == RunState::Idle); +} diff --git a/test/PreferencesSizeGBTest.cpp b/test/PreferencesSizeGBTest.cpp new file mode 100644 index 0000000000..ed9af5bab2 --- /dev/null +++ b/test/PreferencesSizeGBTest.cpp @@ -0,0 +1,474 @@ +#include "simplnx/Core/Preferences.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace nx::core; +namespace fs = std::filesystem; + +namespace +{ +// Expected values are hand calculations with 1 GB = 1024^3 = 1073741824 bytes. +constexpr uint64 k_OneHundredthBytes = 10737418; // floor(1073741824 / 100 + 0.5) +constexpr uint64 k_OneAndAHalfGBBytes = 1610612736; // 1.5 * 1073741824 +constexpr uint64 k_MaxGridBytes = 18446744073698814198ULL; // 17179869183.99 GB +constexpr uint64 k_MaxHundredths = 1717986918399; // 17179869183.99 GB +constexpr int32 k_LegacyRepairCode = -591; +constexpr int32 k_GBRepairCode = -592; + +/** + * @struct SizeGBFiles + * @brief Owns a unique temporary directory with an input and an output preferences file. + */ +struct SizeGBFiles +{ + fs::path root; + fs::path input; + fs::path output; + + SizeGBFiles() + { + const auto stamp = std::chrono::steady_clock::now().time_since_epoch().count(); + for(usize attempt = 0; attempt < 1000; ++attempt) + { + const auto candidate = fs::temp_directory_path() / ("simplnx_size_gb_preferences_" + std::to_string(stamp) + "_" + std::to_string(attempt)); + if(fs::create_directory(candidate)) + { + root = candidate; + input = root / "input.json"; + output = root / "output.json"; + return; + } + } + throw std::runtime_error("Could not create a unique temporary directory for GB preference tests."); + } + + ~SizeGBFiles() + { + std::error_code errorCode; + fs::remove_all(root, errorCode); + } + + SizeGBFiles(const SizeGBFiles&) = delete; + SizeGBFiles(SizeGBFiles&&) = delete; + SizeGBFiles& operator=(const SizeGBFiles&) = delete; + SizeGBFiles& operator=(SizeGBFiles&&) = delete; +}; + +void writeText(const fs::path& path, const std::string& text) +{ + std::ofstream stream(path, std::ios::binary); + REQUIRE(stream.is_open()); + stream << text; + stream.close(); + REQUIRE_FALSE(stream.fail()); +} + +std::string readText(const fs::path& path) +{ + std::ifstream stream(path, std::ios::binary); + REQUIRE(stream.is_open()); + std::stringstream buffer; + buffer << stream.rdbuf(); + return buffer.str(); +} + +/** + * @brief Loads a preferences document from a new file into a fresh Preferences object. + * @param files Temporary file owner. + * @param document JSON document to write. + * @param preferences Receives the loaded values. + * @return Load result. + */ +Result<> loadDocument(const SizeGBFiles& files, const nlohmann::json& document, Preferences& preferences) +{ + writeText(files.input, document.dump()); + return preferences.loadFromFile(files.input); +} + +const std::array k_ByteKeys = {std::string(Preferences::k_LargeDataSize_Key), std::string(Preferences::k_LargeDataStructureSize_Key)}; +const std::array k_GBKeys = {std::string(Preferences::k_LargeDataSizeGB_Key), std::string(Preferences::k_LargeDataStructureSizeGB_Key)}; +const std::array k_Labels = {"Large Data Size", "Large Data Structure Size"}; +} // namespace + +TEST_CASE("Preferences OOC size grid converts between bytes and hundredths of a GB", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + CHECK(Preferences::k_BytesPerGiB == 1073741824); + CHECK(Preferences::k_MaxOocSizeHundredths == k_MaxHundredths); + + CHECK(Preferences::OocSizeHundredthsToBytes(0) == 0); + CHECK(Preferences::OocSizeHundredthsToBytes(1) == k_OneHundredthBytes); + CHECK(Preferences::OocSizeHundredthsToBytes(100) == 1073741824); + CHECK(Preferences::OocSizeHundredthsToBytes(150) == k_OneAndAHalfGBBytes); + CHECK(Preferences::OocSizeHundredthsToBytes(k_MaxHundredths) == k_MaxGridBytes); + // A value above the domain converts as the maximum. + CHECK(Preferences::OocSizeHundredthsToBytes(k_MaxHundredths + 1) == k_MaxGridBytes); + CHECK(Preferences::OocSizeHundredthsToBytes(std::numeric_limits::max()) == k_MaxGridBytes); + + CHECK(Preferences::OocSizeBytesToHundredths(0) == 0); + CHECK(Preferences::OocSizeBytesToHundredths(1073741824) == 100); + CHECK(Preferences::OocSizeBytesToHundredths(123456789) == 11); + CHECK(Preferences::OocSizeBytesToHundredths(k_MaxGridBytes) == k_MaxHundredths); + CHECK(Preferences::OocSizeBytesToHundredths(std::numeric_limits::max()) == k_MaxHundredths); + // A positive byte count never becomes 0 GB. + CHECK(Preferences::OocSizeBytesToHundredths(1) == 1); + CHECK(Preferences::OocSizeBytesToHundredths(10000) == 1); + // 0.005 GB is 5368709.12 bytes: 5368709 rounds down to 0 (then becomes 1); 5368710 rounds up to 1. + CHECK(Preferences::OocSizeBytesToHundredths(5368709) == 1); + CHECK(Preferences::OocSizeBytesToHundredths(5368710) == 1); + // 1.005 GB is 1079110533.12 bytes. + CHECK(Preferences::OocSizeBytesToHundredths(1079110533) == 100); + CHECK(Preferences::OocSizeBytesToHundredths(1079110534) == 101); + + for(const uint64 hundredths : {uint64{0}, uint64{1}, uint64{99}, uint64{100}, uint64{101}, k_MaxHundredths - 1, k_MaxHundredths}) + { + CAPTURE(hundredths); + CHECK(Preferences::OocSizeBytesToHundredths(Preferences::OocSizeHundredthsToBytes(hundredths)) == hundredths); + } + usize mismatches = 0; + for(uint64 hundredths = 0; hundredths <= 200000; ++hundredths) + { + mismatches += Preferences::OocSizeBytesToHundredths(Preferences::OocSizeHundredthsToBytes(hundredths)) == hundredths ? 0 : 1; + } + constexpr uint64 k_Stride = 17179869; // About 100000 samples across the domain. + for(uint64 hundredths = 7; hundredths <= k_MaxHundredths; hundredths += k_Stride) + { + mismatches += Preferences::OocSizeBytesToHundredths(Preferences::OocSizeHundredthsToBytes(hundredths)) == hundredths ? 0 : 1; + } + CHECK(mismatches == 0); +} + +TEST_CASE("Preferences loads valid GB thresholds on the 0.01-GB grid", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + struct ValidCase + { + const char* description; + nlohmann::json value; + uint64 expectedBytes; + }; + const std::vector cases = {{"integer 1", nlohmann::json(int64{1}), 1073741824}, + {"unsigned 2", nlohmann::json(uint64{2}), 2147483648}, + {"1.5", nlohmann::json(1.5), k_OneAndAHalfGBBytes}, + {"integer 0", nlohmann::json(int64{0}), 0}, + {"0.0", nlohmann::json(0.0), 0}, + {"-0.0", nlohmann::json(-0.0), 0}, + {"0.01", nlohmann::json(0.01), k_OneHundredthBytes}, + {"0.001", nlohmann::json(0.001), k_OneHundredthBytes}, + {"0.004", nlohmann::json(0.004), k_OneHundredthBytes}, + {"13.85", nlohmann::json(13.85), 14871324262}, + {"maximum", nlohmann::json(17179869183.99), k_MaxGridBytes}}; + for(usize keyIndex = 0; keyIndex < k_GBKeys.size(); ++keyIndex) + { + for(const auto& validCase : cases) + { + DYNAMIC_SECTION(k_GBKeys[keyIndex] << "=" << validCase.description) + { + SizeGBFiles files; + Preferences preferences; + const auto result = loadDocument(files, {{k_GBKeys[keyIndex], validCase.value}, {"unrelated", "preserved"}}, preferences); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + REQUIRE(preferences.contains(k_ByteKeys[keyIndex])); + const auto stored = preferences.value(k_ByteKeys[keyIndex]); + REQUIRE(stored.is_number_unsigned()); + CHECK(stored.get() == validCase.expectedBytes); + CHECK_FALSE(preferences.contains(k_GBKeys[keyIndex])); + CHECK_FALSE(preferences.contains(k_ByteKeys[1 - keyIndex])); + CHECK(preferences.value("unrelated") == nlohmann::json("preserved")); + } + } + } +} + +TEST_CASE("Preferences repairs invalid GB thresholds with their own warning", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + struct InvalidCase + { + const char* description; + nlohmann::json value; + std::string shownValue; + std::string shownType; + }; + const std::vector cases = {{"negative integer", nlohmann::json(int64{-1}), "-1", "number_integer"}, + {"negative float", nlohmann::json(-0.5), "-0.5", "number_float"}, + {"string", nlohmann::json("1.5"), "\"1.5\"", "string"}, + {"boolean", nlohmann::json(true), "true", "boolean"}, + {"null", nlohmann::json(nullptr), "null", "null"}, + {"array", nlohmann::json::array({1}), "[1]", "array"}, + {"1e300", nlohmann::json(1e300), "1e+300", "number_float"}, + {"above maximum", nlohmann::json(17179869184.0), "17179869184.0", "number_float"}, + {"UINT64_MAX", nlohmann::json(std::numeric_limits::max()), "18446744073709551615", "number_unsigned"}}; + for(usize keyIndex = 0; keyIndex < k_GBKeys.size(); ++keyIndex) + { + for(const auto& invalidCase : cases) + { + DYNAMIC_SECTION(k_GBKeys[keyIndex] << "=" << invalidCase.description) + { + SizeGBFiles files; + Preferences preferences; + const uint64 defaultBytes = preferences.defaultValueAs(k_ByteKeys[keyIndex]); + const uint64 defaultHundredths = (defaultBytes / 1073741824) * 100 + ((defaultBytes % 1073741824) * 100 + 536870912) / 1073741824; + const std::string defaultText = std::to_string(defaultHundredths / 100) + "." + (defaultHundredths % 100 < 10 ? "0" : "") + std::to_string(defaultHundredths % 100); + const auto document = nlohmann::json::object({{k_GBKeys[keyIndex], invalidCase.value}, {"unrelated", "preserved"}}); + const auto result = loadDocument(files, document, preferences); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings().front().code == k_GBRepairCode); + const std::string expected = "Preferences: saved " + k_Labels[keyIndex] + " ('" + k_GBKeys[keyIndex] + "') value " + invalidCase.shownValue + " (JSON type " + invalidCase.shownType + + ") is invalid. Allowed domain: 0 to 17179869183.99 GB. Using the default: " + defaultText + " GB. Review this value in Preferences > Out of Core."; + CHECK(result.warnings().front().message == expected); + CHECK_FALSE(preferences.contains(k_ByteKeys[keyIndex])); + CHECK_FALSE(preferences.contains(k_GBKeys[keyIndex])); + CHECK(preferences.valueAs(k_ByteKeys[keyIndex]) == defaultBytes); + CHECK(preferences.value("unrelated") == nlohmann::json("preserved")); + CHECK(nlohmann::json::parse(readText(files.input)) == document); + } + } + } + SECTION("Large Data Size default text") + { + SizeGBFiles files; + Preferences preferences; + const auto result = loadDocument(files, {{k_GBKeys[0], int64{-1}}}, preferences); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings().front().message.find("Using the default: 1.00 GB.") != std::string::npos); + } +} + +TEST_CASE("Preferences converts legacy byte thresholds to the 0.01-GB grid", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + struct LegacyCase + { + const char* description; + nlohmann::json value; + uint64 expectedBytes; + }; + const std::vector cases = {{"0", nlohmann::json(uint64{0}), 0}, + {"10000 bytes", nlohmann::json(int64{10000}), k_OneHundredthBytes}, + {"123456789 bytes", nlohmann::json(uint64{123456789}), 118111601}, + {"1 GiB", nlohmann::json(uint64{1073741824}), 1073741824}, + {"UINT64_MAX", nlohmann::json(std::numeric_limits::max()), k_MaxGridBytes}}; + for(usize keyIndex = 0; keyIndex < k_ByteKeys.size(); ++keyIndex) + { + for(const auto& legacyCase : cases) + { + DYNAMIC_SECTION(k_ByteKeys[keyIndex] << "=" << legacyCase.description) + { + SizeGBFiles files; + Preferences preferences; + const auto result = loadDocument(files, {{k_ByteKeys[keyIndex], legacyCase.value}}, preferences); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + const auto stored = preferences.value(k_ByteKeys[keyIndex]); + REQUIRE(stored.is_number_unsigned()); + CHECK(stored.get() == legacyCase.expectedBytes); + CHECK_FALSE(preferences.contains(k_GBKeys[keyIndex])); + } + } + DYNAMIC_SECTION(k_ByteKeys[keyIndex] << " invalid keeps the legacy warning") + { + SizeGBFiles files; + Preferences preferences; + const uint64 defaultBytes = preferences.defaultValueAs(k_ByteKeys[keyIndex]); + const auto result = loadDocument(files, {{k_ByteKeys[keyIndex], int64{-1}}}, preferences); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings().front().code == k_LegacyRepairCode); + CHECK(result.warnings().front().message == "Preferences: saved " + k_Labels[keyIndex] + " ('" + k_ByteKeys[keyIndex] + + "') value -1 (JSON type number_integer) is invalid. Allowed domain: 0 to 18446744073709551615 bytes. Using the default: " + + std::to_string(defaultBytes) + " bytes. Review this value in Preferences > Out of Core."); + CHECK_FALSE(preferences.contains(k_ByteKeys[keyIndex])); + } + } +} + +TEST_CASE("Preferences gives a saved GB threshold precedence over a byte threshold", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + for(usize keyIndex = 0; keyIndex < k_GBKeys.size(); ++keyIndex) + { + DYNAMIC_SECTION(k_GBKeys[keyIndex] << " valid GB and valid bytes") + { + SizeGBFiles files; + Preferences preferences; + const auto result = loadDocument(files, {{k_GBKeys[keyIndex], 2}, {k_ByteKeys[keyIndex], uint64{1073741824}}}, preferences); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + CHECK(preferences.valueAs(k_ByteKeys[keyIndex]) == 2147483648); + CHECK_FALSE(preferences.contains(k_GBKeys[keyIndex])); + } + DYNAMIC_SECTION(k_GBKeys[keyIndex] << " valid GB and invalid bytes") + { + SizeGBFiles files; + Preferences preferences; + const auto result = loadDocument(files, {{k_GBKeys[keyIndex], 1.5}, {k_ByteKeys[keyIndex], "bad"}}, preferences); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + CHECK(preferences.valueAs(k_ByteKeys[keyIndex]) == k_OneAndAHalfGBBytes); + CHECK_FALSE(preferences.contains(k_GBKeys[keyIndex])); + } + DYNAMIC_SECTION(k_GBKeys[keyIndex] << " invalid GB and valid bytes") + { + SizeGBFiles files; + Preferences preferences; + const uint64 defaultBytes = preferences.defaultValueAs(k_ByteKeys[keyIndex]); + const auto result = loadDocument(files, {{k_GBKeys[keyIndex], "bad"}, {k_ByteKeys[keyIndex], uint64{1073741824 * 3ULL}}}, preferences); + REQUIRE(result.valid()); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings().front().code == k_GBRepairCode); + CHECK(result.warnings().front().message.find(k_GBKeys[keyIndex]) != std::string::npos); + CHECK_FALSE(preferences.contains(k_ByteKeys[keyIndex])); + CHECK_FALSE(preferences.contains(k_GBKeys[keyIndex])); + CHECK(preferences.valueAs(k_ByteKeys[keyIndex]) == defaultBytes); + } + } +} + +TEST_CASE("Preferences GB conversion keeps the legacy cache budget migration", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + SizeGBFiles files; + Preferences preferences; + const auto result = loadDocument(files, {{k_ByteKeys[0], uint64{10000}}, {k_GBKeys[1], 2.5}, {"memory_budget_bytes", uint64{123456}}}, preferences); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + CHECK(preferences.valueAs(k_ByteKeys[0]) == k_OneHundredthBytes); + CHECK(preferences.valueAs(k_ByteKeys[1]) == 2684354560); + CHECK(preferences.cacheMemoryBudgetBytes() == 123456); + CHECK(preferences.contains(Preferences::k_CacheMemoryBudgetBytes_Key)); + CHECK_FALSE(preferences.contains(Preferences::k_LegacyMemoryBudgetBytes_Key)); + REQUIRE(preferences.saveToFile(files.output).valid()); + const auto saved = nlohmann::json::parse(readText(files.output)); + CHECK(saved.at(std::string(Preferences::k_CacheMemoryBudgetBytes_Key)).get() == 123456); + CHECK(saved.at(k_GBKeys[0]).get() == 0.01); + CHECK(saved.at(k_GBKeys[1]).get() == 2.5); +} + +TEST_CASE("Preferences repeated GB loads keep no stale threshold state", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + SizeGBFiles files; + Preferences preferences; + const uint64 defaultBytes = preferences.defaultValueAs(k_ByteKeys[0]); + + const auto failed = loadDocument(files, {{k_GBKeys[0], -3}}, preferences); + REQUIRE(failed.valid()); + REQUIRE(failed.warnings().size() == 1); + CHECK(failed.warnings().front().code == k_GBRepairCode); + CHECK(preferences.valueAs(k_ByteKeys[0]) == defaultBytes); + + const auto good = loadDocument(files, {{k_GBKeys[0], 1.5}}, preferences); + REQUIRE(good.valid()); + CHECK(good.warnings().empty()); + CHECK(preferences.valueAs(k_ByteKeys[0]) == k_OneAndAHalfGBBytes); + + const auto absent = loadDocument(files, nlohmann::json::object({{"plugins", nlohmann::json::object()}}), preferences); + REQUIRE(absent.valid()); + CHECK(absent.warnings().empty()); + CHECK_FALSE(preferences.contains(k_ByteKeys[0])); + CHECK(preferences.valueAs(k_ByteKeys[0]) == defaultBytes); +} + +TEST_CASE("Preferences saves OOC thresholds only as GB keys", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + struct SaveCase + { + uint64 bytes; + std::string text; + }; + const std::vector cases = {{0, "0.0"}, {10000, "0.01"}, {1073741824, "1.0"}, {k_OneAndAHalfGBBytes, "1.5"}, {14871324262, "13.85"}, {std::numeric_limits::max(), "17179869183.99"}}; + for(usize keyIndex = 0; keyIndex < k_ByteKeys.size(); ++keyIndex) + { + for(const auto& saveCase : cases) + { + DYNAMIC_SECTION(k_ByteKeys[keyIndex] << "=" << saveCase.bytes) + { + SizeGBFiles files; + Preferences preferences; + preferences.setValue(k_ByteKeys[keyIndex], saveCase.bytes); + preferences.setValue("unrelated", "preserved"); + preferences.setPluginValue("SizeGBTest", "keep", 17); + // The in-memory value stays exact. Only the file uses the 0.01-GB grid. + CHECK(preferences.valueAs(k_ByteKeys[keyIndex]) == saveCase.bytes); + REQUIRE(preferences.saveToFile(files.output).valid()); + const std::string text = readText(files.output); + CHECK(text.find("\"" + k_GBKeys[keyIndex] + "\":" + saveCase.text) != std::string::npos); + CHECK(text.find("\"" + k_ByteKeys[0] + "\"") == std::string::npos); + CHECK(text.find("\"" + k_ByteKeys[1] + "\"") == std::string::npos); + const auto saved = nlohmann::json::parse(text); + CHECK(saved.at("unrelated") == nlohmann::json("preserved")); + CHECK(saved.at("plugins").at("SizeGBTest").at("keep") == nlohmann::json(17)); + CHECK(preferences.valueAs(k_ByteKeys[keyIndex]) == saveCase.bytes); + } + } + } +} + +TEST_CASE("Preferences converts a legacy file only when it saves", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + SizeGBFiles files; + const std::string legacyText = R"({"large_data_size":10000,"large_datastructure_size":18446744073709551615,"plugins":{},"unrelated":"preserved"})"; + writeText(files.input, legacyText); + const auto modifiedBefore = fs::last_write_time(files.input); + + Preferences preferences; + const auto result = preferences.loadFromFile(files.input); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + CHECK(readText(files.input) == legacyText); + // Evaluate to bool before Catch2 tries to stream libc++'s file-clock time point. + CHECK((fs::last_write_time(files.input) == modifiedBefore)); + CHECK(preferences.valueAs(k_ByteKeys[0]) == k_OneHundredthBytes); + CHECK(preferences.valueAs(k_ByteKeys[1]) == k_MaxGridBytes); + + REQUIRE(preferences.saveToFile(files.input).valid()); + const auto converted = nlohmann::json::parse(readText(files.input)); + CHECK_FALSE(converted.contains(k_ByteKeys[0])); + CHECK_FALSE(converted.contains(k_ByteKeys[1])); + CHECK(converted.at(k_GBKeys[0]).get() == 0.01); + CHECK(converted.at(k_GBKeys[1]).get() == 17179869183.99); + CHECK(converted.at("unrelated") == nlohmann::json("preserved")); +} + +TEST_CASE("Preferences GB save and load round trip keeps exact grid bytes", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + const uint64 ramDefault = Preferences().defaultValueAs(k_ByteKeys[1]); + for(const uint64 bytes : {uint64{0}, k_OneHundredthBytes, uint64{1073741824}, ramDefault, k_MaxGridBytes}) + { + for(usize keyIndex = 0; keyIndex < k_ByteKeys.size(); ++keyIndex) + { + DYNAMIC_SECTION(k_ByteKeys[keyIndex] << "=" << bytes) + { + SizeGBFiles files; + Preferences preferences; + preferences.setValue(k_ByteKeys[keyIndex], bytes); + REQUIRE(preferences.saveToFile(files.output).valid()); + Preferences loaded; + const auto result = loaded.loadFromFile(files.output); + REQUIRE(result.valid()); + CHECK(result.warnings().empty()); + CHECK(loaded.valueAs(k_ByteKeys[keyIndex]) == bytes); + } + } + } +} + +TEST_CASE("Preferences threshold defaults are on the 0.01-GB grid", "[Core][Preferences][OocThreshold][PreferencesSizeGB]") +{ + Preferences preferences; + const uint64 arrayDefault = preferences.defaultValueAs(k_ByteKeys[0]); + const uint64 structureDefault = preferences.defaultValueAs(k_ByteKeys[1]); + CHECK(arrayDefault == 1073741824); + CHECK(Preferences::OocSizeHundredthsToBytes(Preferences::OocSizeBytesToHundredths(arrayDefault)) == arrayDefault); + CHECK(Preferences::OocSizeHundredthsToBytes(Preferences::OocSizeBytesToHundredths(structureDefault)) == structureDefault); + // clear() recomputes the defaults and keeps them on the grid. + preferences.clear(); + CHECK(preferences.defaultValueAs(k_ByteKeys[1]) == structureDefault); +} diff --git a/test/ProfileIsolationContract.cpp b/test/ProfileIsolationContract.cpp new file mode 100644 index 0000000000..73bbe7b26e --- /dev/null +++ b/test/ProfileIsolationContract.cpp @@ -0,0 +1,64 @@ +#include "simplnx/Core/Application.hpp" +#include "simplnx/Core/Preferences.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace +{ +// This deliberate extra owner tests the runner's failure-retention path. +std::shared_ptr s_RetainedApplication; +} // namespace + +TEST_CASE("Test profile saves and reloads an isolated witness", "[.TestProfile]") +{ + namespace fs = std::filesystem; + const char* active = std::getenv("SIMPLNX_TEST_ACTIVE_PROFILE"); + REQUIRE(active != nullptr); + const fs::path root(active); + REQUIRE(root.is_absolute()); + REQUIRE(fs::is_regular_file(root / ".simplnx-test-profile")); + REQUIRE(std::getenv("SIMPLNX_TEST_PROFILE_DIR") == nullptr); + REQUIRE(std::getenv("SIMPLNX_TEST_PROFILE_TOKEN") == nullptr); + REQUIRE(std::getenv("SIMPLNX_TEST_PROFILE_REPORT") == nullptr); +#ifdef _WIN32 + const auto expected = root / "AppData/Local/DREAM3DNX/preferences.json"; +#elif defined(__APPLE__) + const auto expected = root / "Library/Preferences/DREAM3DNX/preferences.json"; +#else + const auto expected = root / ".config/DREAM3DNX/preferences.json"; +#endif + REQUIRE(nx::core::Preferences::DefaultFilePath("DREAM3DNX") == expected); + auto app = nx::core::Application::GetOrCreateInstance(); + app->getPreferences()->setValue("profile_round_trip", 731); + REQUIRE(app->savePreferences().valid()); + nx::core::Preferences loaded; + REQUIRE(loaded.loadFromFile(expected).valid()); + REQUIRE(loaded.value("profile_round_trip") == 731); +} + +TEST_CASE("Test profile retains a failed run", "[.TestProfile]") +{ + REQUIRE(std::getenv("SIMPLNX_TEST_ACTIVE_PROFILE") != nullptr); + FAIL("Deliberate profile retention probe"); +} + +TEST_CASE("Test profile detects a retained Application owner", "[.TestProfile]") +{ + s_RetainedApplication = nx::core::Application::GetOrCreateInstance(); + REQUIRE(s_RetainedApplication != nullptr); +} + +TEST_CASE("Test profile detects an Application save failure", "[.TestProfile]") +{ + auto app = nx::core::Application::GetOrCreateInstance(); + REQUIRE(app != nullptr); + const auto preferences = nx::core::Preferences::DefaultFilePath("DREAM3DNX"); + REQUIRE(std::filesystem::remove(preferences)); + REQUIRE(std::filesystem::create_directory(preferences)); +} diff --git a/test/ProfileIsolationLifecycleProbe.cpp b/test/ProfileIsolationLifecycleProbe.cpp new file mode 100644 index 0000000000..6655d8f8b3 --- /dev/null +++ b/test/ProfileIsolationLifecycleProbe.cpp @@ -0,0 +1,158 @@ +#include "simplnx/UnitTest/ProcessTestProfile.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +namespace fs = std::filesystem; +using nx::core::UnitTest::ProcessTestProfile; + +std::optional environmentValue(const char* name) +{ + const char* value = std::getenv(name); + return value == nullptr ? std::nullopt : std::optional(value); +} + +struct LateWitness +{ + LateWitness() = default; + LateWitness(const LateWitness&) = default; + LateWitness& operator=(const LateWitness&) = default; + + // Rvalues copy the path so the source keeps its shutdown check. + LateWitness(LateWitness&& other) + : LateWitness(static_cast(other)) + { + } + + LateWitness& operator=(LateWitness&& other) + { + return *this = static_cast(other); + } + + fs::path root; + + ~LateWitness() noexcept + { + if(root.empty()) + { + return; + } + try + { + const char* home = std::getenv(ProcessTestProfile::homeVariable()); + if(fs::exists(root) || home == nullptr || fs::path(home) != root) + { + std::_Exit(86); + } + // This static observer uses retained routing without accessing Application. + fs::create_directories(fs::path(home)); + std::ofstream stream(fs::path(home) / "late-witness.txt"); + stream << "isolated after removal"; + stream.close(); + if(!stream) + { + std::_Exit(87); + } + } catch(...) + { + std::_Exit(88); + } + } +}; + +LateWitness s_LateWitness; +} // namespace + +int main(int argc, char* argv[]) +{ + if(argc != 2) + { + return 2; + } + const std::string mode(argv[1]); + const std::array names = {"HOME", "USERPROFILE", "LOCALAPPDATA", "APPDATA", "TEMP", "TMP", "TMPDIR"}; + std::array, 7> before; + for(size_t index = 0; index < names.size(); ++index) + { + before[index] = environmentValue(names[index]); + } + ProcessTestProfile profile; + std::string error; + if(!profile.install(error)) + { + std::cerr << error << '\n'; + return 3; + } + for(size_t index = 0; index < names.size(); ++index) + { + if(std::string(names[index]) != ProcessTestProfile::homeVariable() && environmentValue(names[index]) != before[index]) + { + std::cerr << "Unrelated environment variable changed: " << names[index] << '\n'; + return 4; + } + } + if(std::getenv("SIMPLNX_TEST_PROFILE_DIR") != nullptr || std::getenv("SIMPLNX_TEST_PROFILE_TOKEN") != nullptr || std::getenv("SIMPLNX_TEST_PROFILE_REPORT") != nullptr) + { + return 5; + } + const auto root = profile.root(); + if(mode == "unfinished") + { + return 9; + } + if(mode == "tamper" || mode == "status-256-failure") + { + std::ofstream(root / ".simplnx-test-profile") << "changed-token"; + } + else if(mode == "nested-owner") + { + fs::create_directory(root / "independent"); + std::ofstream(root / "independent/.simplnx-test-profile") << "independent-owner"; + } + else if(mode == "root-replaced") + { + fs::rename(root, root.string() + "-original"); + fs::create_directory(root); + std::ofstream(root / "unrelated.txt") << "replacement must survive"; + } + else if(mode == "late") + { + s_LateWitness.root = root; + } + else if(mode != "success" && mode != "failure" && mode != "status-256-success") + { + return 6; + } + if(mode == "status-256-failure" || mode == "status-256-success") + { + const int result = profile.finishExitCode(256, true, error); + if(!error.empty()) + { + std::cerr << error << '\n'; + } + return result; + } + const bool finished = profile.finish(mode != "failure", error); + const std::string firstError = error; + if(profile.finish(true, error) != finished || error != firstError) + { + return 7; + } + if(!finished) + { + std::cerr << error << '\n'; + return 8; + } + if(profile.ownsRoot() && fs::exists(root)) + { + return 10; + } + return 0; +} diff --git a/test/SelectiveImportReferencesTest.cpp b/test/SelectiveImportReferencesTest.cpp index 0a5924bdd9..4465cd02b8 100644 --- a/test/SelectiveImportReferencesTest.cpp +++ b/test/SelectiveImportReferencesTest.cpp @@ -1,18 +1,44 @@ +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/Common/Types.hpp" #include "simplnx/Core/Application.hpp" +#include "simplnx/Core/Preferences.hpp" #include "simplnx/DataStructure/AttributeMatrix.hpp" +#include "simplnx/DataStructure/DataArray.hpp" #include "simplnx/DataStructure/DataGroup.hpp" #include "simplnx/DataStructure/DataStore.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" #include "simplnx/DataStructure/Geometry/EdgeGeom.hpp" #include "simplnx/DataStructure/Geometry/ImageGeom.hpp" #include "simplnx/DataStructure/Geometry/RectGridGeom.hpp" #include "simplnx/DataStructure/Geometry/TetrahedralGeom.hpp" #include "simplnx/DataStructure/Geometry/TriangleGeom.hpp" #include "simplnx/DataStructure/Geometry/VertexGeom.hpp" +#include "simplnx/DataStructure/IDataArray.hpp" +#include "simplnx/DataStructure/IDataStore.hpp" +#include "simplnx/DataStructure/IO/Generic/IDataStoreFormatResolver.hpp" +#include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" +#include "simplnx/DataStructure/Messaging/DataRemovedMessage.hpp" #include "simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp" +#include "simplnx/Filter/Output.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp" +#include "simplnx/Utilities/Parsing/HDF5/H5Support.hpp" +#include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" #include "simplnx/unit_test/simplnx_test_dirs.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include @@ -139,8 +165,393 @@ std::vector PopulateReferences(DataStructure& dataStructure, IGeo } return links; } + +class GeometryAliasResolver : public IDataStoreFormatResolver +{ +public: + GeometryAliasResolver() = default; + ~GeometryAliasResolver() override = default; + GeometryAliasResolver(const GeometryAliasResolver&) = delete; + GeometryAliasResolver(GeometryAliasResolver&&) = delete; + GeometryAliasResolver& operator=(const GeometryAliasResolver&) = delete; + GeometryAliasResolver& operator=(GeometryAliasResolver&&) = delete; + + mutable std::atomic Calls{0}; + + std::string resolveFormat(const DataStructure& dataStructure, const DataPath& path, [[maybe_unused]] DataType dataType, [[maybe_unused]] uint64 byteSize) const override + { + ++Calls; + const auto geometryPath = path.getParent(); + const auto* geometryPtr = dataStructure.getDataAs(geometryPath); + const auto* verticesPtr = dataStructure.getDataAs(geometryPath.createChildPath("Vertices")); + const auto* facesPtr = dataStructure.getDataAs>(geometryPath.createChildPath("Faces")); + REQUIRE(geometryPtr != nullptr); + REQUIRE(verticesPtr != nullptr); + REQUIRE(facesPtr != nullptr); + CAPTURE(path.toString()); + CHECK(geometryPtr->getVertexListId() == verticesPtr->getId()); + CHECK(geometryPtr->getFaceListId() == facesPtr->getId()); + CHECK(geometryPtr->getVertices() == verticesPtr); + CHECK(geometryPtr->getFaces() == facesPtr); + CHECK(verticesPtr->getStoreType() == IDataStore::StoreType::Empty); + CHECK(facesPtr->getStoreType() == IDataStore::StoreType::Empty); + return {}; + } +}; } // namespace +TEST_CASE("Selective import references: geometry aliases use their own child arrays", "[SelectiveImportReferences][GeometryAliases]") +{ + UnitTest::LoadPlugins(); + const auto application = Application::GetOrCreateInstance(); + const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool bothPlacements = GENERATE(false, true); + const bool reverse = GENERATE(false, true); + const bool warm = GENERATE(false, true); + CAPTURE(static_cast(mode), bothPlacements, reverse, warm); + const std::array expectedVertices{0, 0, 0, 1, 0, 0, 0, 1, 0}; + const std::array expectedFaces{0, 1, 2}; + DataStructure source; + auto* firstPtr = DataGroup::Create(source, "First"); + auto* secondPtr = DataGroup::Create(source, "Second"); + REQUIRE(firstPtr != nullptr); + REQUIRE(secondPtr != nullptr); + auto* meshPtr = TriangleGeom::Create(source, "Mesh", firstPtr->getId()); + REQUIRE(meshPtr != nullptr); + auto vertexStore = std::make_shared>(ShapeType{3}, ShapeType{3}, 0.0F); + auto faceStore = std::make_shared>(ShapeType{1}, ShapeType{3}, IGeometry::MeshIndexType{0}); + const auto writeVertices = vertexStore->copyFromBuffer(0, expectedVertices); + const auto writeFaces = faceStore->copyFromBuffer(0, expectedFaces); + SIMPLNX_RESULT_REQUIRE_VALID(writeVertices); + SIMPLNX_RESULT_REQUIRE_VALID(writeFaces); + auto* verticesPtr = Float32Array::Create(source, "Vertices", vertexStore, meshPtr->getId()); + auto* facesPtr = DataArray::Create(source, "Faces", faceStore, meshPtr->getId()); + REQUIRE(verticesPtr != nullptr); + REQUIRE(facesPtr != nullptr); + meshPtr->setVertices(*verticesPtr); + meshPtr->setFaceList(*facesPtr); + REQUIRE(source.setAdditionalParent(meshPtr->getId(), secondPtr->getId())); + const ImportReferenceFile file(source, "GeometryAliases"); + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + auto handout = cache.fetchNeutralMetadata(file.path); + SIMPLNX_RESULT_REQUIRE_VALID(handout); + const auto* cachedMeshPtr = handout.value().dataStructure.getDataAs(DataPath({"First", "Mesh"})); + REQUIRE(cachedMeshPtr != nullptr); + const auto cachedVertexId = cachedMeshPtr->getVertexListId(); + const auto cachedFaceId = cachedMeshPtr->getFaceListId(); + cache.invalidate(file.path); + if(warm) + { + auto warmResult = cache.fetchNeutralMetadata(file.path); + SIMPLNX_RESULT_REQUIRE_VALID(warmResult); + } + cache.resetStats(); + const std::string selectedParent = reverse ? "First" : "Second"; + std::vector paths{DataPath({selectedParent}), DataPath({selectedParent, "Mesh"}), DataPath({selectedParent, "Mesh", "Vertices"}), DataPath({selectedParent, "Mesh", "Faces"})}; + if(bothPlacements) + { + const std::string otherParent = reverse ? "Second" : "First"; + paths.insert(paths.end(), {DataPath({otherParent}), DataPath({otherParent, "Mesh"}), DataPath({otherParent, "Mesh", "Vertices"}), DataPath({otherParent, "Mesh", "Faces"})}); + } + DataStructure destination; + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const auto result = ImportH5ObjectPathsAction(file.path, paths).apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(resolver->Calls.load() == (bothPlacements ? 4 : 2)); + CHECK(cache.hitCount() == (warm ? 1 : 0)); + CHECK(cache.missCount() == (warm ? 0 : 1)); + for(const auto& parent : {std::string("First"), std::string("Second")}) + { + if(!bothPlacements && parent != selectedParent) + { + CHECK_FALSE(destination.containsData(DataPath({parent}))); + continue; + } + const DataPath meshPath({parent, "Mesh"}); + const auto* importedPtr = destination.getDataAs(meshPath); + const auto* importedVerticesPtr = destination.getDataAs(meshPath.createChildPath("Vertices")); + const auto* importedFacesPtr = destination.getDataAs>(meshPath.createChildPath("Faces")); + REQUIRE(importedPtr != nullptr); + REQUIRE(importedVerticesPtr != nullptr); + REQUIRE(importedFacesPtr != nullptr); + CAPTURE(meshPath.toString()); + CHECK(importedPtr->getVertexListId() == importedVerticesPtr->getId()); + CHECK(importedPtr->getFaceListId() == importedFacesPtr->getId()); + CHECK(importedPtr->getVertices() == importedVerticesPtr); + CHECK(importedPtr->getFaces() == importedFacesPtr); + CHECK(importedVerticesPtr->getTupleShape() == ShapeType{3}); + CHECK(importedVerticesPtr->getComponentShape() == ShapeType{3}); + CHECK(importedFacesPtr->getTupleShape() == ShapeType{1}); + CHECK(importedFacesPtr->getComponentShape() == ShapeType{3}); + if(mode == IDataAction::Mode::Preflight) + { + CHECK(importedVerticesPtr->getIDataStore()->getStoreType() == IDataStore::StoreType::Empty); + CHECK(importedFacesPtr->getIDataStore()->getStoreType() == IDataStore::StoreType::Empty); + } + else + { + std::array actualVertices{}; + std::array actualFaces{}; + const auto readVertices = importedVerticesPtr->getDataStoreRef().copyIntoBuffer(0, actualVertices); + const auto readFaces = importedFacesPtr->getDataStoreRef().copyIntoBuffer(0, actualFaces); + SIMPLNX_RESULT_REQUIRE_VALID(readVertices); + SIMPLNX_RESULT_REQUIRE_VALID(readFaces); + CHECK(actualVertices == expectedVertices); + CHECK(actualFaces == expectedFaces); + } + } + if(bothPlacements) + { + CHECK(destination.getData(DataPath({"First", "Mesh"})) != destination.getData(DataPath({"Second", "Mesh"}))); + for(const auto* namePtr : {"Vertices", "Faces"}) + { + const auto* firstArrayPtr = destination.getDataAs(DataPath({"First", "Mesh", namePtr})); + const auto* secondArrayPtr = destination.getDataAs(DataPath({"Second", "Mesh", namePtr})); + REQUIRE(firstArrayPtr != nullptr); + REQUIRE(secondArrayPtr != nullptr); + CHECK(firstArrayPtr != secondArrayPtr); + CHECK(firstArrayPtr->getId() != secondArrayPtr->getId()); + CHECK(firstArrayPtr->getIDataStore() != secondArrayPtr->getIDataStore()); + } + } + CHECK(cachedMeshPtr->getVertexListId() == cachedVertexId); + CHECK(cachedMeshPtr->getFaceListId() == cachedFaceId); + CHECK(handout.value().dataStructure.getData(DataPath({"First", "Mesh"})) == handout.value().dataStructure.getData(DataPath({"Second", "Mesh"}))); + CHECK(meshPtr->getVertexListId() == verticesPtr->getId()); + CHECK(meshPtr->getFaceListId() == facesPtr->getId()); + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("Selective import references: omitted local alias keeps a selected shared target", "[SelectiveImportReferences][GeometryAliases]") +{ + UnitTest::LoadPlugins(); + const auto application = Application::GetOrCreateInstance(); + const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool selectRoot = GENERATE(false, true); + DataStructure source; + auto* geometryPtr = ImageGeom::Create(source, "Geometry"); + auto* aliasPtr = DataGroup::Create(source, "Alias"); + REQUIRE(geometryPtr != nullptr); + REQUIRE(aliasPtr != nullptr); + auto* cellsPtr = AttributeMatrix::Create(source, "Cells", {1}, geometryPtr->getId()); + REQUIRE(cellsPtr != nullptr); + geometryPtr->setCellData(cellsPtr->getId()); + REQUIRE(source.setAdditionalParent(geometryPtr->getId(), aliasPtr->getId())); + const ImportReferenceFile file(source, "PartialGeometryAlias"); + const DataPath rootPath({"Geometry"}); + const DataPath aliasPath({"Alias", "Geometry"}); + const auto selectedCellsPath = (selectRoot ? rootPath : aliasPath).createChildPath("Cells"); + const auto omittedCellsPath = (selectRoot ? aliasPath : rootPath).createChildPath("Cells"); + DataStructure destination; + const auto result = ImportH5ObjectPathsAction(file.path, {rootPath, DataPath({"Alias"}), aliasPath, selectedCellsPath}).apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(result); + const auto* selectedCellsPtr = destination.getDataAs(selectedCellsPath); + REQUIRE(selectedCellsPtr != nullptr); + CHECK_FALSE(destination.containsData(omittedCellsPath)); + CHECK(destination.getAllDataPaths().size() == 4); + for(const auto& path : {rootPath, aliasPath}) + { + const auto* importedPtr = destination.getDataAs(path); + REQUIRE(importedPtr != nullptr); + CHECK(importedPtr->getCellDataId() == selectedCellsPtr->getId()); + CHECK(destination.getData(importedPtr->getCellDataId()) == selectedCellsPtr); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("Selective import references: nested targets preserve explicit alias rules", "[SelectiveImportReferences][GeometryAliases]") +{ + UnitTest::LoadPlugins(); + enum class Scenario + { + UniqueNested, + AmbiguousNested, + WrongDirectName, + PrefixSibling, + DirectChild + }; + const auto application = Application::GetOrCreateInstance(); + const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const auto scenario = GENERATE(Scenario::UniqueNested, Scenario::AmbiguousNested, Scenario::WrongDirectName, Scenario::PrefixSibling, Scenario::DirectChild); + const bool reverse = GENERATE(false, true); + CAPTURE(static_cast(mode), static_cast(scenario), reverse); + DataStructure source; + auto* sizesPtr = Float32Array::Create(source, "Sizes", std::make_unique>(ShapeType{1}, ShapeType{1}, 7.0F)); + auto* geometryPtr = VertexGeom::Create(source, "Mesh"); + REQUIRE(sizesPtr != nullptr); + REQUIRE(geometryPtr != nullptr); + geometryPtr->setElementSizesId(sizesPtr->getId()); + DataPath expectedPath({"Mesh", "Nested", "Sizes"}); + if(scenario == Scenario::PrefixSibling) + { + auto* siblingPtr = DataGroup::Create(source, "MeshExtra"); + REQUIRE(siblingPtr != nullptr); + REQUIRE(source.setAdditionalParent(sizesPtr->getId(), siblingPtr->getId())); + expectedPath = DataPath({"Sizes"}); + } + else + { + auto* nestedPtr = DataGroup::Create(source, "Nested", geometryPtr->getId()); + REQUIRE(nestedPtr != nullptr); + REQUIRE(source.setAdditionalParent(sizesPtr->getId(), nestedPtr->getId())); + if(scenario == Scenario::AmbiguousNested || scenario == Scenario::DirectChild) + { + auto* otherPtr = DataGroup::Create(source, "Other", geometryPtr->getId()); + REQUIRE(otherPtr != nullptr); + REQUIRE(source.setAdditionalParent(sizesPtr->getId(), otherPtr->getId())); + expectedPath = DataPath({"Sizes"}); + } + if(scenario == Scenario::WrongDirectName) + { + const auto wrongId = CreateReferenceArray(source, "Sizes", geometryPtr->getId()); + REQUIRE(wrongId != sizesPtr->getId()); + } + if(scenario == Scenario::DirectChild) + { + REQUIRE(source.setAdditionalParent(sizesPtr->getId(), geometryPtr->getId())); + expectedPath = DataPath({"Mesh", "Sizes"}); + } + } + REQUIRE(source.getData(DataPath({"Sizes"})) == sizesPtr); + const ImportReferenceFile file(source, "NestedGeometryAliases"); + auto paths = source.getAllDataPaths(); + // Parent-based path enumeration omits the root placement of this shared array. + paths.emplace_back(std::vector{"Sizes"}); + if(reverse) + { + std::ranges::reverse(paths); + } + DataStructure destination; + const auto result = ImportH5ObjectPathsAction(file.path, paths).apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(result); + const auto* importedPtr = destination.getDataAs(DataPath({"Mesh"})); + const auto* expectedPtr = destination.getDataAs(expectedPath); + REQUIRE(importedPtr != nullptr); + REQUIRE(expectedPtr != nullptr); + CHECK(importedPtr->getElementSizesId() == expectedPtr->getId()); + CHECK(importedPtr->getElementSizes() == expectedPtr); + if(mode == IDataAction::Mode::Execute) + { + CHECK((*expectedPtr)[0] == 7.0F); + } + if(scenario == Scenario::WrongDirectName) + { + const auto* wrongPtr = destination.getData(DataPath({"Mesh", "Sizes"})); + REQUIRE(wrongPtr != nullptr); + CHECK(importedPtr->getElementSizesId() != wrongPtr->getId()); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("Selective import references: many geometry aliases keep unique nested targets", "[SelectiveImportReferences][GeometryAliases]") +{ + UnitTest::LoadPlugins(); + constexpr usize k_AliasCount = 128; + const auto application = Application::GetOrCreateInstance(); + const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + DataStructure source; + auto* firstParentPtr = DataGroup::Create(source, "Alias_0"); + REQUIRE(firstParentPtr != nullptr); + auto* geometryPtr = VertexGeom::Create(source, "Mesh", firstParentPtr->getId()); + REQUIRE(geometryPtr != nullptr); + auto* nestedPtr = DataGroup::Create(source, "Nested", geometryPtr->getId()); + REQUIRE(nestedPtr != nullptr); + const auto sizesId = CreateReferenceArray(source, "Sizes", nestedPtr->getId()); + geometryPtr->setElementSizesId(sizesId); + for(usize index = 1; index < k_AliasCount; ++index) + { + auto* parentPtr = DataGroup::Create(source, "Alias_" + std::to_string(index)); + REQUIRE(parentPtr != nullptr); + REQUIRE(source.setAdditionalParent(geometryPtr->getId(), parentPtr->getId())); + } + const ImportReferenceFile file(source, "ManyGeometryAliases"); + DataStructure destination; + const auto result = ImportH5ObjectPathsAction(file.path, source.getAllDataPaths()).apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(destination.getAllDataPaths().size() == 4 * k_AliasCount); + std::set geometryIds; + std::set targetIds; + for(usize index = 0; index < k_AliasCount; ++index) + { + const DataPath geometryPath({"Alias_" + std::to_string(index), "Mesh"}); + const auto* importedPtr = destination.getDataAs(geometryPath); + const auto* targetPtr = destination.getDataAs(geometryPath.createChildPath("Nested").createChildPath("Sizes")); + REQUIRE(importedPtr != nullptr); + REQUIRE(targetPtr != nullptr); + CAPTURE(geometryPath.toString()); + CHECK(importedPtr->getElementSizesId() == targetPtr->getId()); + CHECK(importedPtr->getElementSizes() == targetPtr); + CHECK(geometryIds.insert(importedPtr->getId()).second); + CHECK(targetIds.insert(targetPtr->getId()).second); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +TEST_CASE("Selective import references: geometry alias publication rolls back and retries", "[SelectiveImportReferences][GeometryAliases]") +{ + UnitTest::LoadPlugins(); + const auto application = Application::GetOrCreateInstance(); + const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + DataStructure source; + auto* firstPtr = DataGroup::Create(source, "First"); + auto* secondPtr = DataGroup::Create(source, "Second"); + REQUIRE(firstPtr != nullptr); + REQUIRE(secondPtr != nullptr); + auto* geometryPtr = ImageGeom::Create(source, "Geometry", firstPtr->getId()); + REQUIRE(geometryPtr != nullptr); + auto* cellsPtr = AttributeMatrix::Create(source, "Cells", {1}, geometryPtr->getId()); + REQUIRE(cellsPtr != nullptr); + geometryPtr->setCellData(cellsPtr->getId()); + REQUIRE(source.setAdditionalParent(geometryPtr->getId(), secondPtr->getId())); + const ImportReferenceFile file(source, "GeometryAliasRollback"); + DataStructure destination; + REQUIRE(DataGroup::Create(destination, "First") != nullptr); + REQUIRE(DataGroup::Create(destination, "Second") != nullptr); + auto* markerPtr = DataGroup::Create(destination, "Existing"); + REQUIRE(markerPtr != nullptr); + const auto beforeIds = destination.getAllDataObjectIds(); + const auto beforePaths = destination.getAllDataPaths(); + const auto beforeNextId = destination.getNextId(); + const std::vector selected{DataPath({"First", "Geometry"}), DataPath({"Second", "Geometry"}), DataPath({"First", "Geometry", "Cells"}), DataPath({"Second", "Geometry", "Cells"})}; + const ImportH5ObjectPathsAction action(file.path, selected); + action.setPublicationFaultForTesting(ImportH5ObjectPathsAction::PublicationFaultType::AfterSecondInsert); + const auto failed = action.apply(destination, mode); + REQUIRE(failed.invalid()); + REQUIRE_FALSE(failed.errors().empty()); + CHECK(failed.errors().front().code == -6214); + CHECK(action.publicationFaultConsumedForTesting()); + CHECK(destination.getAllDataObjectIds() == beforeIds); + CHECK(destination.getAllDataPaths() == beforePaths); + CHECK(destination.getNextId() == beforeNextId); + CHECK(destination.getData(DataPath({"Existing"})) == markerPtr); + const auto retained = std::dynamic_pointer_cast(action.retainedOwnerForTesting()); + REQUIRE(retained != nullptr); + CHECK(retained->getDataStructure() == nullptr); + REQUIRE(retained->getCellDataId().has_value()); + CHECK(destination.getData(retained->getCellDataId()) == nullptr); + const auto retried = ImportH5ObjectPathsAction(file.path, selected).apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(retried); + for(const auto* parentPtr : {"First", "Second"}) + { + const auto* importedPtr = destination.getDataAs(DataPath({parentPtr, "Geometry"})); + const auto* importedCellsPtr = destination.getDataAs(DataPath({parentPtr, "Geometry", "Cells"})); + REQUIRE(importedPtr != nullptr); + REQUIRE(importedCellsPtr != nullptr); + CHECK(importedPtr->getCellDataId() == importedCellsPtr->getId()); + CHECK(destination.getData(importedPtr->getCellDataId()) == importedCellsPtr); + } + CHECK(destination.getData(DataPath({"Existing"})) == markerPtr); + UnitTest::CheckArraysInheritTupleDims(destination); +} +#endif + TEST_CASE("Selective import references: overlapping ElementSizes reset", "[SelectiveImportReferences][DataStructure]") { DataStructure source; @@ -192,12 +603,25 @@ TEST_CASE("Selective import references: excluded cell ID cannot alias an ensembl const auto* importedEnsemble = destination.getDataAs(DataPath({"Image", "CellEnsembleData"})); REQUIRE(imported != nullptr); REQUIRE(importedEnsemble != nullptr); - if(mode == IDataAction::Mode::Execute) - { - REQUIRE(importedEnsemble->getId() == cells->getId()); - } + REQUIRE(importedEnsemble->getId() == ensemble->getId()); REQUIRE_FALSE(imported->getCellDataId().has_value()); REQUIRE_FALSE(destination.containsData(DataPath({"Image", "CellData"}))); + REQUIRE(destination.getData(cells->getId()) == nullptr); + const auto importedIds = destination.getAllDataObjectIds(); + REQUIRE(std::find(importedIds.begin(), importedIds.end(), cells->getId()) == importedIds.end()); + const auto importedNextId = destination.getNextId(); + // Reuse the excluded cell ID explicitly so a stale reference would resolve to an unrelated ensemble. + auto* unrelated = AttributeMatrix::Import(destination, "UnrelatedEnsemble", ShapeType{2}, cells->getId(), imported->getId()); + REQUIRE(unrelated != nullptr); + REQUIRE(unrelated->getId() == cells->getId()); + REQUIRE(destination.getData(cells->getId()) == unrelated); + REQUIRE(destination.getData(DataPath({"Image", "UnrelatedEnsemble"})) == unrelated); + REQUIRE_FALSE(imported->getCellDataId().has_value()); + REQUIRE(destination.getData(DataPath({"Image", "CellEnsembleData"})) == importedEnsemble); + REQUIRE(destination.getData(importedEnsemble->getId()) == importedEnsemble); + REQUIRE(importedEnsemble->getId() == ensemble->getId()); + REQUIRE(importedEnsemble->getShape() == ShapeType{2}); + REQUIRE(destination.getNextId() == importedNextId); REQUIRE(imported->getDimensions() == geometry->getDimensions()); auto validation = imported->validate(); SIMPLNX_RESULT_REQUIRE_VALID(validation); @@ -210,6 +634,7 @@ TEST_CASE("Selective import references: field matrix and full selection", "[Sele const auto application = Application::GetOrCreateInstance(); const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool aliases = GENERATE(false, true); const auto type = GENERATE(IGeometry::Type::Image, IGeometry::Type::Vertex, IGeometry::Type::Edge, IGeometry::Type::Triangle, IGeometry::Type::Tetrahedral, IGeometry::Type::RectGrid); DataStructure source; IGeometry* geometry = nullptr; @@ -238,40 +663,59 @@ TEST_CASE("Selective import references: field matrix and full selection", "[Sele } REQUIRE(geometry != nullptr); const auto links = PopulateReferences(source, *geometry); + std::vector geometryPaths{DataPath({"Geometry"})}; + if(aliases) + { + auto* aliasParentPtr = DataGroup::Create(source, "Alias"); + REQUIRE(aliasParentPtr != nullptr); + REQUIRE(source.setAdditionalParent(geometry->getId(), aliasParentPtr->getId())); + geometryPaths.emplace_back(std::vector{"Alias", "Geometry"}); + } const ImportReferenceFile file(source, "FieldMatrix"); // The final case retains all references with overlapping source and destination ID ranges. for(usize omitted = 0; omitted <= links.size(); ++omitted) { - DYNAMIC_SECTION("type " << static_cast(type) << " mode " << static_cast(mode) << " omitted " << omitted) + DYNAMIC_SECTION("type " << static_cast(type) << " mode " << static_cast(mode) << " omitted " << omitted << " aliases " << aliases) { - std::vector paths{DataPath({"Geometry"})}; - for(usize field = 0; field < links.size(); ++field) + auto paths = geometryPaths; + if(aliases) { - if(field != omitted) + paths.emplace_back(std::vector{"Alias"}); + } + for(const auto& geometryPath : geometryPaths) + { + for(usize field = 0; field < links.size(); ++field) { - paths.push_back(links[field].targetPath); + if(field != omitted) + { + paths.push_back(geometryPath.createChildPath(links[field].targetPath.getTargetName())); + } } } DataStructure destination; destination.setNextId(omitted == links.size() ? 3 : 100); auto result = ImportH5ObjectPathsAction(file.path, paths).apply(destination, mode); SIMPLNX_RESULT_REQUIRE_VALID(result); - const auto* imported = destination.getDataAs(DataPath({"Geometry"})); - REQUIRE(imported != nullptr); - for(usize field = 0; field < links.size(); ++field) + for(const auto& geometryPath : geometryPaths) { - CAPTURE(links[field].targetPath.toString()); - if(field == omitted) + const auto* importedPtr = destination.getDataAs(geometryPath); + REQUIRE(importedPtr != nullptr); + for(usize field = 0; field < links.size(); ++field) { - REQUIRE_FALSE(links[field].getId(*imported).has_value()); - REQUIRE_FALSE(destination.containsData(links[field].targetPath)); - } - else - { - const auto* target = destination.getData(links[field].targetPath); - REQUIRE(target != nullptr); - REQUIRE(links[field].getId(*imported) == target->getId()); - REQUIRE(destination.getData(links[field].getId(*imported)) == target); + const auto targetPath = geometryPath.createChildPath(links[field].targetPath.getTargetName()); + CAPTURE(targetPath.toString()); + if(field == omitted) + { + REQUIRE_FALSE(links[field].getId(*importedPtr).has_value()); + REQUIRE_FALSE(destination.containsData(targetPath)); + } + else + { + const auto* targetPtr = destination.getData(targetPath); + REQUIRE(targetPtr != nullptr); + REQUIRE(links[field].getId(*importedPtr) == targetPtr->getId()); + REQUIRE(destination.getData(links[field].getId(*importedPtr)) == targetPtr); + } } } UnitTest::CheckArraysInheritTupleDims(destination); @@ -319,6 +763,7 @@ TEST_CASE("Selective import references: cached selections and shared targets sta { const auto application = Application::GetOrCreateInstance(); const UnitTest::PreferencesSentinel preferences(DataStorageMode::ForceInCore, 0); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); DataStructure source; auto* first = ImageGeom::Create(source, "First"); auto* second = ImageGeom::Create(source, "Second"); @@ -330,12 +775,12 @@ TEST_CASE("Selective import references: cached selections and shared targets sta second->setCellData(cells->getId()); const ImportReferenceFile file(source, "Cache"); auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); - auto cached = cache.fetch(file.path); + auto cached = cache.fetchNeutralMetadata(file.path); SIMPLNX_RESULT_REQUIRE_VALID(cached); const auto initialHits = cache.hitCount(); DataStructure partial; - auto result = ImportH5ObjectPathsAction(file.path, {DataPath({"First"})}).apply(partial, IDataAction::Mode::Preflight); + auto result = ImportH5ObjectPathsAction(file.path, {DataPath({"First"})}).apply(partial, mode); SIMPLNX_RESULT_REQUIRE_VALID(result); const auto* partialFirst = partial.getDataAs(DataPath({"First"})); REQUIRE(partialFirst != nullptr); @@ -344,7 +789,7 @@ TEST_CASE("Selective import references: cached selections and shared targets sta DataStructure full; full.setNextId(100); - result = ImportH5ObjectPathsAction(file.path, {DataPath({"First"}), DataPath({"Second"}), DataPath({"Cells"})}).apply(full, IDataAction::Mode::Preflight); + result = ImportH5ObjectPathsAction(file.path, {DataPath({"First"}), DataPath({"Second"}), DataPath({"Cells"})}).apply(full, mode); SIMPLNX_RESULT_REQUIRE_VALID(result); REQUIRE(cache.hitCount() >= initialHits + 2); const auto* importedFirst = full.getDataAs(DataPath({"First"})); @@ -357,7 +802,7 @@ TEST_CASE("Selective import references: cached selections and shared targets sta REQUIRE(importedSecond->getCellDataId() == importedCells->getId()); REQUIRE(full.getData(importedFirst->getCellDataId()) == importedCells); REQUIRE(full.getData(importedSecond->getCellDataId()) == importedCells); - const auto* unchanged = cached.value().getDataAs(DataPath({"First"})); + const auto* unchanged = cached.value().dataStructure.getDataAs(DataPath({"First"})); REQUIRE(unchanged != nullptr); REQUIRE(unchanged->getCellDataId() == cells->getId()); UnitTest::CheckArraysInheritTupleDims(partial); @@ -493,3 +938,648 @@ TEST_CASE("Selective import references: selection errors and group shells stay c UnitTest::CheckArraysInheritTupleDims(destination); } } + +namespace +{ +constexpr std::array k_AliasValues{7, -3, 0, 41}; +constexpr std::array k_LaterAliasValues{901, -112}; + +struct AliasPlanRecord +{ + DataPath path; + DataType type; + uint64 bytes = 0; +}; + +class AliasPathResolver : public IDataStoreFormatResolver +{ +public: + AliasPathResolver() = default; + ~AliasPathResolver() override = default; + AliasPathResolver(const AliasPathResolver&) = default; + AliasPathResolver(AliasPathResolver&&) = default; + AliasPathResolver& operator=(const AliasPathResolver&) = default; + AliasPathResolver& operator=(AliasPathResolver&&) = default; + + mutable std::vector records; + + std::string resolveFormat(const DataStructure&, const DataPath& path, DataType type, uint64 bytes) const override + { + records.push_back({path, type, bytes}); + return {}; + } +}; + +template +Int32Array* CreateAliasArray(DataStructure& structure, const std::string& name, DataObject::IdType parent, const std::array& values) +{ + auto store = std::make_shared>(ShapeType{Count}, ShapeType{1}, int32{0}); + for(usize valueIdx = 0; valueIdx < Count; ++valueIdx) + { + store->setValue(valueIdx, values[valueIdx]); + } + auto* array = Int32Array::Create(structure, name, std::move(store), parent); + REQUIRE(array != nullptr); + return array; +} + +void RequireAliasParents(const DataObject& object, std::vector expected) +{ + const auto parents = object.getParentIds(); + std::vector actual(parents.begin(), parents.end()); + std::sort(actual.begin(), actual.end()); + std::sort(expected.begin(), expected.end()); + REQUIRE(actual == expected); +} + +/** + * @brief Validates the closed source through a fixed-size independent HDF5 read. + * @tparam Count Fixes the permitted value count and buffer size. + * @param file Supplies an independently reopened source. + * @param path Identifies the numeric dataset below DataStructure. + * @param expected Supplies the literal value oracle. + */ +template +void RequireAliasFileValues(const HDF5::FileIO& file, const DataPath& path, const std::array& expected) +{ + const auto fileId = file.getId(); + const std::lock_guard lock(HDF5::Support::ApiLock()); + const auto datasetPath = "/DataStructure/" + path.toString(); + const hid_t dataset = H5Dopen2(fileId, datasetPath.c_str(), H5P_DEFAULT); + REQUIRE(dataset >= 0); + const auto datasetGuard = MakeScopeGuard([dataset]() noexcept { H5Dclose(dataset); }); + const hid_t space = H5Dget_space(dataset); + REQUIRE(space >= 0); + const auto spaceGuard = MakeScopeGuard([space]() noexcept { H5Sclose(space); }); + REQUIRE(H5Sget_simple_extent_ndims(space) == 2); + REQUIRE(H5Sget_simple_extent_npoints(space) == static_cast(Count)); + std::array shape{}; + REQUIRE(H5Sget_simple_extent_dims(space, shape.data(), nullptr) == 2); + REQUIRE(shape == std::array{Count, 1}); + const hid_t type = H5Dget_type(dataset); + REQUIRE(type >= 0); + const auto typeGuard = MakeScopeGuard([type]() noexcept { H5Tclose(type); }); + REQUIRE(H5Tequal(type, H5T_NATIVE_INT32) > 0); + std::array actual{}; + REQUIRE(H5Dread(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, actual.data()) >= 0); + REQUIRE(actual == expected); +} + +template +const Int32Array* RequireAliasOutput(const DataStructure& structure, const DataPath& path, IDataAction::Mode mode, const std::array& values) +{ + const auto* array = structure.getDataAs(path); + REQUIRE(array != nullptr); + REQUIRE(structure.getData(array->getId()) == array); + REQUIRE(array->getTupleShape() == ShapeType{Count}); + REQUIRE(array->getComponentShape() == ShapeType{1}); + REQUIRE(array->size() == Count); + REQUIRE(array->getIDataStore() != nullptr); + CHECK(array->getDataFormat().empty()); + CHECK(array->getIDataStore()->getPlannedStoreType() == IDataStore::StoreType::InMemory); + if(mode == IDataAction::Mode::Preflight) + { + CHECK(array->getStoreType() == IDataStore::StoreType::Empty); + } + else + { + REQUIRE(array->getStoreType() == IDataStore::StoreType::InMemory); + for(usize valueIdx = 0; valueIdx < Count; ++valueIdx) + { + CHECK((*array)[valueIdx] == values[valueIdx]); + } + } + return array; +} + +void RequireAliasPlans(const AliasPathResolver& resolver, const std::vector>& expected) +{ + REQUIRE(resolver.records.size() == expected.size()); + for(const auto& expectedRecord : expected) + { + CAPTURE(expectedRecord.first.toString()); + CHECK(std::count_if(resolver.records.begin(), resolver.records.end(), [&](const AliasPlanRecord& record) { return record.path == expectedRecord.first; }) == 1); + } + for(const auto& record : resolver.records) + { + const auto expectedRecord = std::find_if(expected.begin(), expected.end(), [&](const auto& item) { return item.first == record.path; }); + REQUIRE(expectedRecord != expected.end()); + CHECK(record.type == DataType::int32); + CHECK(record.bytes == expectedRecord->second); + } +} + +void RequireAliasPathIds(const DataStructure& structure, const std::vector& expectedPaths) +{ + REQUIRE(structure.getAllDataPaths().size() == expectedPaths.size()); + const auto ids = structure.getAllDataObjectIds(); + REQUIRE(ids.size() == expectedPaths.size()); + std::set seen; + for(const auto& path : expectedPaths) + { + const auto* object = structure.getData(path); + REQUIRE(object != nullptr); + CHECK(structure.getData(object->getId()) == object); + CHECK(seen.insert(object->getId()).second); + } +} +} // namespace + +TEST_CASE("C8 distinct paths to a shared array retain independent destination copies", "[C8][SelectedImportAliases][SelectiveImportReferences]") +{ + const auto application = Application::GetOrCreateInstance(); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool warm = GENERATE(false, true); + CAPTURE(static_cast(mode), warm); + const auto memory = std::make_shared(); + const DataPath firstPath({"First"}); + const DataPath secondPath({"Second"}); + const DataPath firstArrayPath({"First", "Shared"}); + const DataPath secondArrayPath({"Second", "Shared"}); + const DataPath laterGroupPath({"Second", "LaterGroup"}); + const DataPath laterArrayPath({"Second", "LaterGroup", "Later"}); + DataStructure source; + source.setFormatResolver(memory); + auto* first = DataGroup::Create(source, "First"); + auto* second = DataGroup::Create(source, "Second"); + REQUIRE(first != nullptr); + REQUIRE(second != nullptr); + auto* shared = CreateAliasArray(source, "Shared", first->getId(), k_AliasValues); + REQUIRE(source.setAdditionalParent(shared->getId(), second->getId())); + // The deeper distinct object must keep its reserved source ID when an earlier alias is split. + auto* laterGroup = DataGroup::Create(source, "LaterGroup", second->getId()); + REQUIRE(laterGroup != nullptr); + auto* later = CreateAliasArray(source, "Later", laterGroup->getId(), k_LaterAliasValues); + REQUIRE(first->getId() == 1); + REQUIRE(second->getId() == 2); + REQUIRE(shared->getId() == 3); + REQUIRE(laterGroup->getId() == 4); + REQUIRE(later->getId() == 5); + REQUIRE(source.getData(firstArrayPath) == source.getData(secondArrayPath)); + RequireAliasParents(*shared, {first->getId(), second->getId()}); + const ImportReferenceFile file(source, "SharedArrayAliases"); + { + const auto reader = HDF5::FileIO::ReadFile(file.path); + REQUIRE(reader.isValid()); + RequireAliasFileValues(reader, firstArrayPath, k_AliasValues); + RequireAliasFileValues(reader, secondArrayPath, k_AliasValues); + RequireAliasFileValues(reader, laterArrayPath, k_LaterAliasValues); + } + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + auto metadata = cache.fetchNeutralMetadata(file.path); + SIMPLNX_RESULT_REQUIRE_VALID(metadata); + REQUIRE(metadata.value().fileVersion == DREAM3D::k_CurrentFileVersion); + const auto& sourceHandout = metadata.value().dataStructure; + const auto* sourceFirst = sourceHandout.getDataAs(firstArrayPath); + const auto* sourceSecond = sourceHandout.getDataAs(secondArrayPath); + REQUIRE(sourceFirst != nullptr); + REQUIRE(sourceFirst == sourceSecond); + REQUIRE(sourceFirst->getId() == shared->getId()); + REQUIRE(sourceHandout.getData(sourceFirst->getId()) == sourceFirst); + RequireAliasParents(*sourceFirst, {first->getId(), second->getId()}); + const auto sourceNext = sourceHandout.getNextId(); + REQUIRE(sourceNext == 6); + REQUIRE(sourceHandout.getAllDataObjectIds().size() == 5); + REQUIRE(sourceHandout.getAllDataPaths().size() == 6); + const auto* sourceLater = sourceHandout.getDataAs(laterArrayPath); + REQUIRE(sourceLater != nullptr); + REQUIRE(sourceLater->getId() == later->getId()); + REQUIRE(sourceHandout.getData(sourceLater->getId()) == sourceLater); + // Evaluate to bool before Catch2 tries to stream libc++'s 128-bit file-clock duration. + REQUIRE((fs::file_time_type::clock::now() - fs::last_write_time(file.path) >= DREAM3D::Dream3dPreflightCache::k_MtimeTrustWindow)); + cache.invalidate(file.path); + cache.resetStats(); + DataStructure warmDestination; + warmDestination.setFormatResolver(memory); + if(warm) + { + // A valid single-placement action warms the same metadata mode before the alias regression is measured. + const auto warmResult = ImportH5ObjectPathsAction(file.path, {firstPath, firstArrayPath}).apply(warmDestination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(warmResult); + RequireAliasOutput(warmDestination, firstArrayPath, mode, k_AliasValues); + REQUIRE(warmDestination.getAllDataPaths().size() == 2); + REQUIRE(cache.missCount() == 1); + REQUIRE(cache.hitCount() == 0); + cache.resetStats(); + } + REQUIRE(cache.missCount() == 0); + REQUIRE(cache.hitCount() == 0); + DataStructure destination; + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const auto initialNextId = destination.getNextId(); + const std::vector selection{firstPath, secondPath, firstArrayPath, secondArrayPath, laterGroupPath, laterArrayPath}; + const auto result = ImportH5ObjectPathsAction(file.path, selection).apply(destination, mode); + CHECK(cache.missCount() == (warm ? 0 : 1)); + CHECK(cache.hitCount() == (warm ? 1 : 0)); + if(result.invalid()) + { + CHECK(destination.getAllDataPaths().empty()); + CHECK(destination.getAllDataObjectIds().empty()); + CHECK(destination.getNextId() == initialNextId); + } + SIMPLNX_RESULT_REQUIRE_VALID(result); + RequireAliasPathIds(destination, selection); + const auto* firstCopy = RequireAliasOutput(destination, firstArrayPath, mode, k_AliasValues); + const auto* secondCopy = RequireAliasOutput(destination, secondArrayPath, mode, k_AliasValues); + const auto* laterCopy = RequireAliasOutput(destination, laterArrayPath, mode, k_LaterAliasValues); + CHECK(firstCopy != secondCopy); + CHECK(firstCopy->getId() != secondCopy->getId()); + CHECK(firstCopy->getIDataStore() != secondCopy->getIDataStore()); + CHECK(std::min(firstCopy->getId(), secondCopy->getId()) == shared->getId()); + CHECK(std::max(firstCopy->getId(), secondCopy->getId()) == sourceNext); + CHECK(laterCopy->getId() == later->getId()); + REQUIRE(destination.getData(laterGroupPath) != nullptr); + CHECK(destination.getData(laterGroupPath)->getId() == laterGroup->getId()); + CHECK(destination.getNextId() == sourceNext + 1); + RequireAliasParents(*firstCopy, {destination.getData(firstPath)->getId()}); + RequireAliasParents(*secondCopy, {destination.getData(secondPath)->getId()}); + RequireAliasParents(*laterCopy, {destination.getData(laterGroupPath)->getId()}); + RequireAliasPlans(*resolver, {{firstArrayPath, 16}, {secondArrayPath, 16}, {laterArrayPath, 8}}); + CHECK(sourceHandout.getData(firstArrayPath) == sourceHandout.getData(secondArrayPath)); + CHECK(sourceHandout.getData(firstArrayPath)->getId() == shared->getId()); + CHECK(sourceHandout.getNextId() == sourceNext); + if(warm) + { + const auto* warmArray = RequireAliasOutput(warmDestination, firstArrayPath, mode, k_AliasValues); + CHECK(warmArray != firstCopy); + CHECK(warmArray->getIDataStore() != firstCopy->getIDataStore()); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("C8 shared group paths retain independent destination groups and children", "[C8][SelectedImportAliases][SelectiveImportReferences]") +{ + const auto application = Application::GetOrCreateInstance(); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool warm = GENERATE(false, true); + CAPTURE(static_cast(mode), warm); + const auto memory = std::make_shared(); + const DataPath firstPath({"First"}); + const DataPath secondPath({"Second"}); + const DataPath firstGroupPath({"First", "SharedGroup"}); + const DataPath secondGroupPath({"Second", "SharedGroup"}); + const DataPath firstChildPath({"First", "SharedGroup", "Child"}); + const DataPath secondChildPath({"Second", "SharedGroup", "Child"}); + DataStructure source; + source.setFormatResolver(memory); + auto* first = DataGroup::Create(source, "First"); + auto* second = DataGroup::Create(source, "Second"); + REQUIRE(first != nullptr); + REQUIRE(second != nullptr); + auto* sharedGroup = DataGroup::Create(source, "SharedGroup", first->getId()); + REQUIRE(sharedGroup != nullptr); + REQUIRE(source.setAdditionalParent(sharedGroup->getId(), second->getId())); + auto* child = CreateAliasArray(source, "Child", sharedGroup->getId(), k_AliasValues); + REQUIRE(first->getId() == 1); + REQUIRE(second->getId() == 2); + REQUIRE(sharedGroup->getId() == 3); + REQUIRE(child->getId() == 4); + REQUIRE(source.getData(firstGroupPath) == source.getData(secondGroupPath)); + REQUIRE(source.getData(firstChildPath) == source.getData(secondChildPath)); + RequireAliasParents(*sharedGroup, {first->getId(), second->getId()}); + // The child has one direct parent and inherits two paths through that parent's placements. + RequireAliasParents(*child, {sharedGroup->getId()}); + const ImportReferenceFile file(source, "SharedGroupAliases"); + { + const auto reader = HDF5::FileIO::ReadFile(file.path); + REQUIRE(reader.isValid()); + RequireAliasFileValues(reader, firstChildPath, k_AliasValues); + RequireAliasFileValues(reader, secondChildPath, k_AliasValues); + } + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + auto metadata = cache.fetchNeutralMetadata(file.path); + SIMPLNX_RESULT_REQUIRE_VALID(metadata); + const auto& sourceHandout = metadata.value().dataStructure; + const auto* sourceGroup = sourceHandout.getDataAs(firstGroupPath); + const auto* sourceChild = sourceHandout.getDataAs(firstChildPath); + REQUIRE(sourceGroup != nullptr); + REQUIRE(sourceChild != nullptr); + REQUIRE(sourceGroup == sourceHandout.getData(secondGroupPath)); + REQUIRE(sourceChild == sourceHandout.getData(secondChildPath)); + REQUIRE(sourceGroup->getId() == sharedGroup->getId()); + REQUIRE(sourceChild->getId() == child->getId()); + RequireAliasParents(*sourceGroup, {first->getId(), second->getId()}); + RequireAliasParents(*sourceChild, {sourceGroup->getId()}); + const auto sourceNext = sourceHandout.getNextId(); + REQUIRE(sourceNext == 5); + REQUIRE(sourceHandout.getAllDataObjectIds().size() == 4); + REQUIRE(sourceHandout.getAllDataPaths().size() == 6); + // Evaluate to bool before Catch2 tries to stream libc++'s 128-bit file-clock duration. + REQUIRE((fs::file_time_type::clock::now() - fs::last_write_time(file.path) >= DREAM3D::Dream3dPreflightCache::k_MtimeTrustWindow)); + cache.invalidate(file.path); + cache.resetStats(); + DataStructure warmDestination; + warmDestination.setFormatResolver(memory); + if(warm) + { + const auto warmResult = ImportH5ObjectPathsAction(file.path, {firstPath, firstGroupPath, firstChildPath}).apply(warmDestination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(warmResult); + RequireAliasOutput(warmDestination, firstChildPath, mode, k_AliasValues); + REQUIRE(warmDestination.getAllDataPaths().size() == 3); + REQUIRE(cache.missCount() == 1); + REQUIRE(cache.hitCount() == 0); + cache.resetStats(); + } + REQUIRE(cache.missCount() == 0); + REQUIRE(cache.hitCount() == 0); + DataStructure destination; + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const auto initialNextId = destination.getNextId(); + const std::vector selection{firstPath, secondPath, firstGroupPath, secondGroupPath, firstChildPath, secondChildPath}; + const auto result = ImportH5ObjectPathsAction(file.path, selection).apply(destination, mode); + CHECK(cache.missCount() == (warm ? 0 : 1)); + CHECK(cache.hitCount() == (warm ? 1 : 0)); + if(result.invalid()) + { + CHECK(destination.getAllDataPaths().empty()); + CHECK(destination.getAllDataObjectIds().empty()); + CHECK(destination.getNextId() == initialNextId); + } + SIMPLNX_RESULT_REQUIRE_VALID(result); + RequireAliasPathIds(destination, selection); + const auto* firstGroup = destination.getDataAs(firstGroupPath); + const auto* secondGroup = destination.getDataAs(secondGroupPath); + REQUIRE(firstGroup != nullptr); + REQUIRE(secondGroup != nullptr); + CHECK(firstGroup != secondGroup); + CHECK(firstGroup->getId() != secondGroup->getId()); + CHECK(std::min(firstGroup->getId(), secondGroup->getId()) == sharedGroup->getId()); + CHECK(std::max(firstGroup->getId(), secondGroup->getId()) == sourceNext); + CHECK(firstGroup->getSize() == 1); + CHECK(secondGroup->getSize() == 1); + const auto* firstChild = RequireAliasOutput(destination, firstChildPath, mode, k_AliasValues); + const auto* secondChild = RequireAliasOutput(destination, secondChildPath, mode, k_AliasValues); + CHECK(firstChild != secondChild); + CHECK(firstChild->getId() != secondChild->getId()); + CHECK(firstChild->getIDataStore() != secondChild->getIDataStore()); + CHECK(std::min(firstChild->getId(), secondChild->getId()) == child->getId()); + CHECK(std::max(firstChild->getId(), secondChild->getId()) == sourceNext + 1); + CHECK(destination.getNextId() == sourceNext + 2); + RequireAliasParents(*firstGroup, {destination.getData(firstPath)->getId()}); + RequireAliasParents(*secondGroup, {destination.getData(secondPath)->getId()}); + RequireAliasParents(*firstChild, {firstGroup->getId()}); + RequireAliasParents(*secondChild, {secondGroup->getId()}); + RequireAliasPlans(*resolver, {{firstChildPath, 16}, {secondChildPath, 16}}); + CHECK(sourceHandout.getData(firstGroupPath) == sourceHandout.getData(secondGroupPath)); + CHECK(sourceHandout.getData(firstChildPath) == sourceHandout.getData(secondChildPath)); + CHECK(sourceHandout.getData(firstGroupPath)->getId() == sharedGroup->getId()); + CHECK(sourceHandout.getData(firstChildPath)->getId() == child->getId()); + if(warm) + { + const auto* warmChild = RequireAliasOutput(warmDestination, firstChildPath, mode, k_AliasValues); + CHECK(warmChild != firstChild); + CHECK(warmChild->getIDataStore() != firstChild->getIDataStore()); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +namespace +{ +DataStructure MakeAliasControlSource() +{ + DataStructure source; + source.setFormatResolver(std::make_shared()); + auto* first = DataGroup::Create(source, "First"); + auto* second = DataGroup::Create(source, "Second"); + REQUIRE(first != nullptr); + REQUIRE(second != nullptr); + auto* shared = CreateAliasArray(source, "Shared", first->getId(), k_AliasValues); + REQUIRE(source.setAdditionalParent(shared->getId(), second->getId())); + REQUIRE(source.getAllDataObjectIds().size() == 3); + REQUIRE(source.getNextId() == 4); + REQUIRE(source.getData(DataPath({"First", "Shared"})) == source.getData(DataPath({"Second", "Shared"}))); + RequireAliasParents(*shared, {first->getId(), second->getId()}); + return source; +} + +const std::vector k_AliasControlSelection{DataPath({"First"}), DataPath({"Second"}), DataPath({"First", "Shared"}), DataPath({"Second", "Shared"})}; + +void RequireAliasControlFile(const ImportReferenceFile& file) +{ + const auto reader = HDF5::FileIO::ReadFile(file.path); + REQUIRE(reader.isValid()); + RequireAliasFileValues(reader, DataPath({"First", "Shared"}), k_AliasValues); + RequireAliasFileValues(reader, DataPath({"Second", "Shared"}), k_AliasValues); +} +} // namespace + +TEST_CASE("C8 alias identifier capacity includes every extra placement", "[C8][SelectedImportAliases]") +{ + const auto application = Application::GetOrCreateInstance(); + const bool fits = GENERATE(false, true); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + CAPTURE(fits, static_cast(mode)); + auto source = MakeAliasControlSource(); + const ImportReferenceFile file(source, "AliasIdentifierCapacity"); + RequireAliasControlFile(file); + constexpr auto maximumId = std::numeric_limits::max(); + const auto startId = maximumId - (fits ? 4 : 3); + DataStructure destination; + destination.setNextId(startId); + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const auto result = ImportH5ObjectPathsAction(file.path, k_AliasControlSelection).apply(destination, mode); + if(!fits) + { + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -6213); + CHECK(result.errors()[0].message.find("additional placement count 1") != std::string::npos); + CHECK(result.errors()[0].message.find(std::to_string(startId)) != std::string::npos); + CHECK(destination.getNextId() == startId); + CHECK(destination.getAllDataPaths().empty()); + CHECK(destination.getAllDataObjectIds().empty()); + CHECK(resolver->records.empty()); + } + else + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + RequireAliasPathIds(destination, k_AliasControlSelection); + const auto* first = RequireAliasOutput(destination, DataPath({"First", "Shared"}), mode, k_AliasValues); + const auto* second = RequireAliasOutput(destination, DataPath({"Second", "Shared"}), mode, k_AliasValues); + CHECK(first != second); + CHECK(first->getIDataStore() != second->getIDataStore()); + CHECK(std::min(first->getId(), second->getId()) == maximumId - 2); + CHECK(std::max(first->getId(), second->getId()) == maximumId - 1); + CHECK(destination.getNextId() == maximumId); + RequireAliasPlans(*resolver, {{DataPath({"First", "Shared"}), 16}, {DataPath({"Second", "Shared"}), 16}}); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +TEST_CASE("C8 alias clone IDs preserve live expired and hierarchy-only occupants", "[C8][SelectedImportAliases]") +{ + const auto application = Application::GetOrCreateInstance(); + const int occupantKind = GENERATE(0, 1, 2); + CAPTURE(occupantKind); + auto source = MakeAliasControlSource(); + const ImportReferenceFile file(source, "AliasOccupiedCloneId"); + RequireAliasControlFile(file); + const auto cloneId = source.getNextId(); + REQUIRE(cloneId == 4); + DataStructure destination; + std::unique_ptr retainedFile; + std::unique_ptr retainingAction; + const DataPath occupiedPath = occupantKind == 2 ? DataPath({"RetainedParent", "RetainedCloneId"}) : DataPath({"OccupiedCloneId"}); + if(occupantKind == 2) + { + // The existing high-ID parent keeps the retained child outside the registry without relying on sibling order. + auto* retainedParent = DataGroup::Import(destination, "RetainedParent", 100); + REQUIRE(retainedParent != nullptr); + destination.setNextId(1); + DataStructure retentionSource; + auto* sourceParent = DataGroup::Create(retentionSource, "RetainedParent"); + REQUIRE(sourceParent != nullptr); + REQUIRE(DataGroup::Create(retentionSource, "Unselected") != nullptr); + REQUIRE(DataGroup::Create(retentionSource, "Earlier") != nullptr); + auto* retained = DataGroup::Create(retentionSource, "RetainedCloneId", sourceParent->getId()); + REQUIRE(retained != nullptr); + REQUIRE(retained->getId() == cloneId); + retainedFile = std::make_unique(retentionSource, "AliasHierarchyOnlyId"); + retainingAction = std::make_unique(retainedFile->path, std::vector{DataPath({"Earlier"}), occupiedPath}); + retainingAction->setPublicationFaultForTesting(ImportH5ObjectPathsAction::PublicationFaultType::AfterHierarchy, true); + const auto retainedResult = retainingAction->apply(destination, IDataAction::Mode::Preflight); + REQUIRE(retainedResult.invalid()); + REQUIRE(retainingAction->publicationFaultConsumedForTesting()); + REQUIRE(retainingAction->cleanupFaultConsumedForTesting()); + REQUIRE(destination.getData(occupiedPath) != nullptr); + REQUIRE(destination.getData(occupiedPath)->getId() == cloneId); + REQUIRE(destination.getData(cloneId) == nullptr); + REQUIRE(destination.getAllDataObjectIds() == std::vector{100}); + REQUIRE(destination.getData(DataPath({"RetainedParent"})) == retainedParent); + const auto entry = retainedParent->getDataMap().find(cloneId); + REQUIRE(entry != retainedParent->getDataMap().end()); + REQUIRE(entry->second.get() == destination.getData(occupiedPath)); + } + else + { + auto* occupied = DataGroup::Import(destination, "OccupiedCloneId", cloneId); + REQUIRE(occupied != nullptr); + REQUIRE(occupied->getId() == cloneId); + if(occupantKind == 1) + { + REQUIRE(destination.removeData(cloneId)); + REQUIRE(destination.getData(cloneId) == nullptr); + REQUIRE(destination.getAllDataObjectIds() == std::vector{cloneId}); + } + } + const auto* originalOwner = destination.getData(occupiedPath); + const auto beforeIds = destination.getAllDataObjectIds(); + const auto beforePaths = destination.getAllDataPaths(); + destination.setNextId(1); + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const auto result = ImportH5ObjectPathsAction(file.path, k_AliasControlSelection).apply(destination, IDataAction::Mode::Preflight); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -6213); + CHECK(result.errors()[0].message.find("identifier 4 conflicts with an existing or selected object") != std::string::npos); + CHECK(resolver->records.empty()); + CHECK(destination.getData(occupiedPath) == originalOwner); + CHECK(destination.getAllDataObjectIds() == beforeIds); + CHECK(destination.getAllDataPaths() == beforePaths); + CHECK(destination.getNextId() == 1); + if(retainingAction != nullptr) + { + const auto cleaned = retainingAction->cleanupFailedImportForTesting(destination); + SIMPLNX_RESULT_REQUIRE_VALID(cleaned); + CHECK(destination.getData(occupiedPath) == nullptr); + const auto* parent = destination.getDataAs(DataPath({"RetainedParent"})); + REQUIRE(parent != nullptr); + CHECK(parent->empty()); + CHECK(destination.getData(100) == parent); + } +} + +TEST_CASE("C8 alias publication rollback removes both copies and restores identifiers", "[C8][SelectedImportAliases][SelectedImportPublication]") +{ + const auto application = Application::GetOrCreateInstance(); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + CAPTURE(static_cast(mode)); + auto source = MakeAliasControlSource(); + const ImportReferenceFile file(source, "AliasPublicationRollback"); + RequireAliasControlFile(file); + const DataPath firstPath({"First"}); + const DataPath secondPath({"Second"}); + const DataPath firstArrayPath({"First", "Shared"}); + const DataPath secondArrayPath({"Second", "Shared"}); + const DataPath markerPath({"First", "Existing"}); + DataStructure destination; + auto* firstParent = DataGroup::Create(destination, "First"); + auto* secondParent = DataGroup::Create(destination, "Second"); + REQUIRE(firstParent != nullptr); + REQUIRE(secondParent != nullptr); + auto* marker = CreateAliasArray(destination, "Existing", firstParent->getId(), k_LaterAliasValues); + const auto* markerStore = marker->getIDataStore(); + const auto beforeIds = destination.getAllDataObjectIds(); + const auto beforePaths = destination.getAllDataPaths(); + const auto beforeNext = destination.getNextId(); + const auto originalCopyId = beforeNext + 2; + const auto cloneId = beforeNext + 3; + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + std::vector removals; + nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr& message) { + if(message->getMsgType() == DataRemovedMessage::MsgType) + { + removals.push_back(std::static_pointer_cast(message)->getId()); + } + })); + const std::vector leaves{firstArrayPath, secondArrayPath}; + ImportH5ObjectPathsAction action(file.path, leaves); + action.setPublicationFaultForTesting(ImportH5ObjectPathsAction::PublicationFaultType::AfterSecondInsert); + const auto result = action.apply(destination, mode); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors()[0].code == -6214); + REQUIRE(action.publicationFaultConsumedForTesting()); + CHECK(destination.getAllDataObjectIds() == beforeIds); + CHECK(destination.getAllDataPaths() == beforePaths); + CHECK(destination.getNextId() == beforeNext); + CHECK(destination.getData(firstPath) == firstParent); + CHECK(destination.getData(secondPath) == secondParent); + CHECK(firstParent->getSize() == 1); + CHECK(secondParent->empty()); + CHECK(destination.getData(markerPath) == marker); + CHECK(marker->getIDataStore() == markerStore); + RequireAliasOutput(destination, markerPath, IDataAction::Mode::Execute, k_LaterAliasValues); + RequireAliasPlans(*resolver, {{firstArrayPath, 16}, {secondArrayPath, 16}}); + std::sort(removals.begin(), removals.end()); + CHECK(removals == std::vector{originalCopyId, cloneId}); + auto retained = action.retainedOwnerForTesting(); + REQUIRE(retained != nullptr); + CHECK(retained->getId() == cloneId); + CHECK(retained->getDataStructure() == nullptr); + resolver->records.clear(); + const ImportH5ObjectPathsAction retry(file.path, leaves); + const auto retried = retry.apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(retried); + const auto* firstCopy = RequireAliasOutput(destination, firstArrayPath, mode, k_AliasValues); + const auto* secondCopy = RequireAliasOutput(destination, secondArrayPath, mode, k_AliasValues); + CHECK(firstCopy != secondCopy); + CHECK(firstCopy->getIDataStore() != secondCopy->getIDataStore()); + CHECK(std::min(firstCopy->getId(), secondCopy->getId()) == originalCopyId); + CHECK(std::max(firstCopy->getId(), secondCopy->getId()) == cloneId); + CHECK(destination.getData(cloneId) != retained.get()); + RequireAliasParents(*firstCopy, {firstParent->getId()}); + RequireAliasParents(*secondCopy, {secondParent->getId()}); + RequireAliasPlans(*resolver, {{firstArrayPath, 16}, {secondArrayPath, 16}}); + const auto removalCount = removals.size(); + retained.reset(); + action.setPublicationFaultForTesting(ImportH5ObjectPathsAction::PublicationFaultType::None); + CHECK(removals.size() == removalCount); + const auto nextId = destination.getNextId(); + REQUIRE(nextId == cloneId + 1); + auto* next = DataGroup::Create(destination, "AfterAliasRetry"); + REQUIRE(next != nullptr); + CHECK(next->getId() == nextId); + CHECK(destination.getData(nextId) == next); + CHECK(destination.getData(DataPath({"AfterAliasRetry"})) == next); + CHECK(destination.getData(markerPath) == marker); + CHECK(marker->getIDataStore() == markerStore); + UnitTest::CheckArraysInheritTupleDims(destination); +} +#endif diff --git a/test/StorageFormatPlanTest.cpp b/test/StorageFormatPlanTest.cpp index 03fec7a26b..f5451503cd 100644 --- a/test/StorageFormatPlanTest.cpp +++ b/test/StorageFormatPlanTest.cpp @@ -1,22 +1,34 @@ +#include "simplnx/Common/Result.hpp" #include "simplnx/Common/ScopeGuard.hpp" #include "simplnx/Core/Application.hpp" #include "simplnx/Core/Preferences.hpp" +#include "simplnx/DataStructure/BaseGroup.hpp" #include "simplnx/DataStructure/DataArray.hpp" +#include "simplnx/DataStructure/DataGroup.hpp" #include "simplnx/DataStructure/DataStructure.hpp" #include "simplnx/DataStructure/EmptyDataStore.hpp" +#include "simplnx/DataStructure/EmptyListStore.hpp" #include "simplnx/DataStructure/IO/Generic/DataIOCollection.hpp" #include "simplnx/DataStructure/IO/Generic/IDataIOManager.hpp" #include "simplnx/DataStructure/IO/Generic/IDataStoreFormatResolver.hpp" #include "simplnx/DataStructure/IO/Generic/IOConstants.hpp" #include "simplnx/DataStructure/IO/Generic/InMemoryFormatResolver.hpp" +#include "simplnx/DataStructure/IO/HDF5/DataStoreIO.hpp" +#include "simplnx/DataStructure/IO/HDF5/DataStructureReader.hpp" +#include "simplnx/DataStructure/Messaging/DataAddedMessage.hpp" +#include "simplnx/DataStructure/Messaging/DataRemovedMessage.hpp" +#include "simplnx/DataStructure/NeighborList.hpp" +#include "simplnx/DataStructure/StringArray.hpp" #include "simplnx/Filter/Actions/CreateArrayAction.hpp" #include "simplnx/Filter/Actions/ImportH5ObjectPathsAction.hpp" +#include "simplnx/UnitTest/ScopedDefaultFormatResolver.hpp" #include "simplnx/UnitTest/UnitTestCommon.hpp" #include "simplnx/Utilities/DataStoreUtilities.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dPreflightCache.hpp" #include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" #include "simplnx/Utilities/StoreCopyUtilities.hpp" +#include #if defined(SIMPLNX_STORE_COPY_STRICT_FORMAT) && SIMPLNX_STORE_COPY_STRICT_FORMAT #include "SimplnxOoc/OocDataIOManager.hpp" @@ -31,12 +43,15 @@ #include #include +#include +#include #include #include #include #include #include #include +#include #include #include #include @@ -500,8 +515,1274 @@ TEST_CASE("StorageFormatPlan: import resolver receives 100 GB logical bytes from auto actionApplyResult = action.apply(destination, IDataAction::Mode::Preflight); SIMPLNX_RESULT_REQUIRE_VALID(actionApplyResult); - REQUIRE(cache.missCount() == 1); - REQUIRE(cache.hitCount() == 1); + // Ordinary and neutral metadata use independent masters. + REQUIRE(cache.missCount() == 2); + REQUIRE(cache.hitCount() == 0); REQUIRE(recordingResolver->paths == std::vector{k_LargeArrayPath}); REQUIRE(recordingResolver->logicalBytes == std::vector{k_LogicalBytes}); } + +namespace +{ +const DataPath k_ImportPolicyGroup({"SelectedGroup"}); +const DataPath k_ImportPolicyFirst({"SelectedGroup", "First"}); +const DataPath k_ImportPolicySecond({"SelectedGroup", "Second"}); +const DataPath k_ImportPolicyExcluded({"SelectedGroup", "Excluded"}); +const DataPath k_ImportPolicyMarker({"ExistingMarker"}); +constexpr std::array k_ImportPolicyFirstValues{3, -7, 11, 29, 0, 101}; +constexpr std::array k_ImportPolicySecondValues{-9, 42, 5, 18}; +constexpr std::array k_ImportPolicyExcludedValues{91, 92, 93}; +constexpr std::array k_ImportPolicyMarkerValues{731, -204}; + +template +Int32Array* CreateImportPolicyArray(DataStructure& dataStructure, const DataPath& path, const std::array& values, DataObject::OptionalId parentId = {}) +{ + auto store = DataStoreUtilities::CreateDataStore(dataStructure, path, ShapeType{Count}, ShapeType{1}); + REQUIRE(store != nullptr); + auto* array = Int32Array::Create(dataStructure, path.getTargetName(), std::move(store), parentId); + REQUIRE(array != nullptr); + for(usize valueIdx = 0; valueIdx < Count; ++valueIdx) + { + (*array)[valueIdx] = values[valueIdx]; + } + return array; +} + +/** + * @brief Checks the closed fixture through a fixed-size native HDF5 read. + * @tparam Count Specifies the exact permitted dataset size. + * @param fileReader Supplies the independently reopened source file. + * @param path Identifies the numeric dataset below DataStructure. + * @param expected Supplies literal values independent of import output. + */ +template +void CheckImportPolicyFileArray(const HDF5::FileIO& fileReader, const DataPath& path, const std::array& expected) +{ + const auto fileId = fileReader.getId(); + const std::lock_guard lock(HDF5::Support::ApiLock()); + const std::string datasetPath = "/DataStructure/" + path.toString(); + const hid_t datasetId = H5Dopen2(fileId, datasetPath.c_str(), H5P_DEFAULT); + REQUIRE(datasetId >= 0); + const auto datasetGuard = MakeScopeGuard([datasetId]() noexcept { H5Dclose(datasetId); }); + const hid_t spaceId = H5Dget_space(datasetId); + REQUIRE(spaceId >= 0); + const auto spaceGuard = MakeScopeGuard([spaceId]() noexcept { H5Sclose(spaceId); }); + REQUIRE(H5Sget_simple_extent_ndims(spaceId) == 2); + REQUIRE(H5Sget_simple_extent_npoints(spaceId) == static_cast(Count)); + std::array dimensions{}; + REQUIRE(H5Sget_simple_extent_dims(spaceId, dimensions.data(), nullptr) == 2); + REQUIRE(dimensions == std::array{Count, 1}); + std::array actual{}; + REQUIRE(H5Dread(datasetId, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, actual.data()) >= 0); + REQUIRE(actual == expected); +} + +void WriteImportPolicyFile(const fs::path& path) +{ + DataStructure source; + source.setFormatResolver(std::make_shared()); + auto* group = DataGroup::Create(source, k_ImportPolicyGroup.getTargetName()); + REQUIRE(group != nullptr); + CreateImportPolicyArray(source, k_ImportPolicyFirst, k_ImportPolicyFirstValues, group->getId()); + CreateImportPolicyArray(source, k_ImportPolicySecond, k_ImportPolicySecondValues, group->getId()); + CreateImportPolicyArray(source, k_ImportPolicyExcluded, k_ImportPolicyExcludedValues, group->getId()); + const auto writeResult = DREAM3D::WriteFile(path, source); + SIMPLNX_RESULT_REQUIRE_VALID(writeResult); + { + const auto fileReader = HDF5::FileIO::ReadFile(path); + REQUIRE(fileReader.isValid()); + CheckImportPolicyFileArray(fileReader, k_ImportPolicyFirst, k_ImportPolicyFirstValues); + CheckImportPolicyFileArray(fileReader, k_ImportPolicySecond, k_ImportPolicySecondValues); + CheckImportPolicyFileArray(fileReader, k_ImportPolicyExcluded, k_ImportPolicyExcludedValues); + } + fs::last_write_time(path, fs::file_time_type::clock::now() - std::chrono::seconds(10)); + REQUIRE((fs::file_time_type::clock::now() - fs::last_write_time(path) >= DREAM3D::Dream3dPreflightCache::k_MtimeTrustWindow)); +} + +template +void CheckImportPolicyArray(const DataStructure& dataStructure, const DataPath& path, const std::array& expected, IDataAction::Mode mode) +{ + const auto* array = dataStructure.getDataAs(path); + REQUIRE(array != nullptr); + CHECK(dataStructure.getData(array->getId()) == array); + REQUIRE(array->getTupleShape() == ShapeType{Count}); + REQUIRE(array->getComponentShape() == ShapeType{1}); + REQUIRE(array->size() == Count); + REQUIRE(array->getIDataStore() != nullptr); + CHECK(array->getDataFormat().empty()); + CHECK(array->getIDataStore()->getPlannedStoreType() == IDataStore::StoreType::InMemory); + if(mode == IDataAction::Mode::Preflight) + { + REQUIRE(array->getStoreType() == IDataStore::StoreType::Empty); + CHECK(array->memoryUsage() == Count * sizeof(int32)); + } + else + { + REQUIRE(array->getStoreType() == IDataStore::StoreType::InMemory); + for(usize valueIdx = 0; valueIdx < Count; ++valueIdx) + { + CHECK((*array)[valueIdx] == expected[valueIdx]); + } + } +} + +void CheckImportPolicyOutput(const DataStructure& dataStructure, IDataAction::Mode mode) +{ + const auto* group = dataStructure.getDataAs(k_ImportPolicyGroup); + REQUIRE(group != nullptr); + CHECK(dataStructure.getData(group->getId()) == group); + CheckImportPolicyArray(dataStructure, k_ImportPolicyFirst, k_ImportPolicyFirstValues, mode); + CheckImportPolicyArray(dataStructure, k_ImportPolicySecond, k_ImportPolicySecondValues, mode); + CHECK_FALSE(dataStructure.containsData(k_ImportPolicyExcluded)); + UnitTest::CheckArraysInheritTupleDims(dataStructure); +} + +void CheckImportPolicyRecords(const RecordingFormatResolver& resolver) +{ + REQUIRE(resolver.paths.size() == 2); + REQUIRE(resolver.numericTypes.size() == 2); + REQUIRE(resolver.logicalBytes.size() == 2); + CHECK(std::count(resolver.paths.begin(), resolver.paths.end(), k_ImportPolicyFirst) == 1); + CHECK(std::count(resolver.paths.begin(), resolver.paths.end(), k_ImportPolicySecond) == 1); + for(usize recordIdx = 0; recordIdx < resolver.paths.size(); ++recordIdx) + { + CHECK(resolver.numericTypes[recordIdx] == DataType::int32); + const uint64 expectedBytes = resolver.paths[recordIdx] == k_ImportPolicyFirst ? sizeof(int32) * k_ImportPolicyFirstValues.size() : sizeof(int32) * k_ImportPolicySecondValues.size(); + CHECK(resolver.logicalBytes[recordIdx] == expectedBytes); + } +} + +void CheckImportPolicyMarker(const DataStructure& destination, const std::shared_ptr& markerOwner, const IDataStore* originalStore) +{ + const auto* marker = destination.getDataAs(k_ImportPolicyMarker); + REQUIRE(marker != nullptr); + CHECK(marker == markerOwner.get()); + CHECK(destination.getData(markerOwner->getId()) == markerOwner.get()); + CHECK(marker->getIDataStore() == originalStore); + const auto tag = marker->getMetadata().getData("policy-marker"); + const auto* tagValue = std::any_cast(&tag); + REQUIRE(tagValue != nullptr); + CHECK(*tagValue == "keep-existing-metadata"); + CheckImportPolicyArray(destination, k_ImportPolicyMarker, k_ImportPolicyMarkerValues, IDataAction::Mode::Execute); +} + +class RejectingImportProcessResolver : public IDataStoreFormatResolver +{ +public: + RejectingImportProcessResolver() = default; + ~RejectingImportProcessResolver() override = default; + RejectingImportProcessResolver(const RejectingImportProcessResolver&) = default; + RejectingImportProcessResolver(RejectingImportProcessResolver&&) = default; + RejectingImportProcessResolver& operator=(const RejectingImportProcessResolver&) = default; + RejectingImportProcessResolver& operator=(RejectingImportProcessResolver&&) = default; + + mutable usize calls = 0; + + std::string resolveFormat(const DataStructure&, const DataPath&, DataType, uint64) const override + { + ++calls; + throw std::runtime_error("selected-import process policy witness"); + } +}; + +/** + * @class CompleteImportContextResolver + * @brief Records complete selected metadata without changing the resident storage choice. + */ +class CompleteImportContextResolver : public RecordingFormatResolver +{ +public: + CompleteImportContextResolver() = default; + ~CompleteImportContextResolver() override = default; + CompleteImportContextResolver(const CompleteImportContextResolver&) = default; + CompleteImportContextResolver(CompleteImportContextResolver&&) = default; + CompleteImportContextResolver& operator=(const CompleteImportContextResolver&) = default; + CompleteImportContextResolver& operator=(CompleteImportContextResolver&&) = default; + + mutable usize completeContextCalls = 0; + mutable usize placeholderContextCalls = 0; + + std::string resolveFormat(const DataStructure& dataStructure, const DataPath& path, DataType type, uint64 bytes) const override + { + const auto* first = dataStructure.getDataAs(k_ImportPolicyFirst); + const auto* second = dataStructure.getDataAs(k_ImportPolicySecond); + if(dataStructure.containsData(k_ImportPolicyMarker) && dataStructure.getDataAs(k_ImportPolicyGroup) != nullptr && first != nullptr && second != nullptr && + !dataStructure.containsData(k_ImportPolicyExcluded)) + { + ++completeContextCalls; + } + if(first != nullptr && second != nullptr && first->getStoreType() == IDataStore::StoreType::Empty && second->getStoreType() == IDataStore::StoreType::Empty) + { + ++placeholderContextCalls; + } + return RecordingFormatResolver::resolveFormat(dataStructure, path, type, bytes); + } +}; + +/** + * @brief Checks the saved owner and the next automatic storage-format decision. + * @param expectedResolver Supplies the owner that must remain active. + * @param expectedFormat Supplies the independently selected format name. + */ +void CheckSelectedImportDefaultResolver(const std::shared_ptr& expectedResolver, const std::string& expectedFormat) +{ + const auto actualResolver = DataStructure::getDefaultFormatResolver(); + CHECK(actualResolver == expectedResolver); + CHECK_FALSE(actualResolver.owner_before(expectedResolver)); + CHECK_FALSE(expectedResolver.owner_before(actualResolver)); + const DataStructure laterStructure; + CHECK(&laterStructure.formatResolver() == expectedResolver.get()); + const auto format = ResolveNumericStorageFormat(laterStructure, DataPath({"LaterArray"}), DataType::int32, sizeof(int32), ""); + SIMPLNX_RESULT_REQUIRE_VALID(format); + CHECK(format.value() == expectedFormat); +} +} // namespace + +TEST_CASE("StorageFormatPlan: selected import does not consult process policy", "[StorageFormatPlan][SelectedImportPolicy]") +{ + UnitTest::LoadPlugins(); + const auto registration = Application::GetOrCreateInstance()->getIOCollection().addIOManager(std::make_shared()); + SIMPLNX_RESULT_REQUIRE_VALID(registration); + const auto incomingResolver = std::make_shared(k_SelectedFormat.str()); + const auto restoreIncoming = UnitTest::SetDefaultFormatResolverForScope(incomingResolver); + { + const auto application = Application::GetOrCreateInstance(); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + const bool warmCache = GENERATE(false, true); + CAPTURE(static_cast(mode), warmCache); + const ScopedTempFile file(fs::path(unit_test::k_BinaryTestOutputDir.view()) / "selected_import_process_policy.dream3d"); + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + const auto memoryResolver = std::make_shared(); + const auto restoreResolver = UnitTest::SetDefaultFormatResolverForScope(memoryResolver); + const auto cleanup = MakeScopeGuard([&cache]() noexcept { cache.clear(); }); + cache.clear(); + cache.resetStats(); + WriteImportPolicyFile(file.path); + const ImportH5ObjectPathsAction action(file.path, {k_ImportPolicySecond, k_ImportPolicyGroup, k_ImportPolicyFirst}); + DataStructure warmDestination; + warmDestination.setFormatResolver(memoryResolver); + if(warmCache) + { + // The action warms its own metadata mode. An ordinary fetch can use a different cache entry. + const auto warmResult = action.apply(warmDestination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(warmResult); + CheckImportPolicyOutput(warmDestination, mode); + REQUIRE(cache.missCount() == 1); + REQUIRE(cache.hitCount() == 0); + cache.resetStats(); + } + REQUIRE(cache.missCount() == 0); + REQUIRE(cache.hitCount() == 0); + + DataStructure destination; + auto* markerPtr = CreateImportPolicyArray(destination, k_ImportPolicyMarker, k_ImportPolicyMarkerValues); + markerPtr->getMetadata().setData("policy-marker", std::string("keep-existing-metadata")); + const auto markerOwner = destination.getSharedData(k_ImportPolicyMarker); + const auto* markerStorePtr = markerPtr->getIDataStore(); + const auto nextId = destination.getNextId(); + const auto destinationResolver = std::make_shared(); + destination.setFormatResolver(destinationResolver); + const auto processResolver = std::make_shared(); + DataStructure::setDefaultFormatResolver(processResolver); + + const auto result = action.apply(destination, mode); + INFO("Process-policy callbacks: " << processResolver->calls); + CHECK(processResolver->calls == 0); + CHECK(cache.missCount() == (warmCache ? 0 : 1)); + CHECK(cache.hitCount() == (warmCache ? 1 : 0)); + CHECK(&destination.formatResolver() == destinationResolver.get()); + CheckImportPolicyMarker(destination, markerOwner, markerStorePtr); + if(result.invalid()) + { + CHECK(destination.getAllDataPaths() == std::vector{k_ImportPolicyMarker}); + CHECK(destination.getNextId() == nextId); + } + SIMPLNX_RESULT_REQUIRE_VALID(result); + CheckImportPolicyRecords(*destinationResolver); + CheckImportPolicyOutput(destination, mode); + CHECK(destination.getAllDataPaths().size() == 4); + if(warmCache) + { + for(const auto& path : {k_ImportPolicyFirst, k_ImportPolicySecond}) + { + const auto* firstHandoutPtr = warmDestination.getDataAs(path); + const auto* secondHandoutPtr = destination.getDataAs(path); + REQUIRE(firstHandoutPtr != nullptr); + REQUIRE(secondHandoutPtr != nullptr); + CHECK(firstHandoutPtr != secondHandoutPtr); + CHECK(firstHandoutPtr->getIDataStore() != secondHandoutPtr->getIDataStore()); + } + CheckImportPolicyOutput(warmDestination, mode); + } + } + CheckSelectedImportDefaultResolver(incomingResolver, k_SelectedFormat.str()); +} + +TEST_CASE("StorageFormatPlan: destination policy sees all selected metadata before values", "[StorageFormatPlan][SelectedImportPolicy]") +{ + UnitTest::LoadPlugins(); + const auto registration = Application::GetOrCreateInstance()->getIOCollection().addIOManager(std::make_shared()); + SIMPLNX_RESULT_REQUIRE_VALID(registration); + const auto incomingResolver = std::make_shared(k_SelectedFormat.str()); + const auto restoreIncoming = UnitTest::SetDefaultFormatResolverForScope(incomingResolver); + { + const auto application = Application::GetOrCreateInstance(); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + CAPTURE(static_cast(mode)); + const ScopedTempFile file(fs::path(unit_test::k_BinaryTestOutputDir.view()) / "selected_import_complete_context.dream3d"); + auto& cache = DREAM3D::Dream3dPreflightCache::Instance(); + const auto memoryResolver = std::make_shared(); + const auto restoreResolver = UnitTest::SetDefaultFormatResolverForScope(memoryResolver); + const auto cleanup = MakeScopeGuard([&cache]() noexcept { cache.clear(); }); + cache.clear(); + WriteImportPolicyFile(file.path); + + DataStructure destination; + auto* markerPtr = CreateImportPolicyArray(destination, k_ImportPolicyMarker, k_ImportPolicyMarkerValues); + markerPtr->getMetadata().setData("policy-marker", std::string("keep-existing-metadata")); + const auto markerOwner = destination.getSharedData(k_ImportPolicyMarker); + const auto* markerStorePtr = markerPtr->getIDataStore(); + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const ImportH5ObjectPathsAction action(file.path, {k_ImportPolicySecond, k_ImportPolicyGroup, k_ImportPolicyFirst}); + const auto result = action.apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CheckImportPolicyRecords(*resolver); + CHECK(resolver->completeContextCalls == 2); + CHECK(resolver->placeholderContextCalls == 2); + CHECK(&destination.formatResolver() == resolver.get()); + CheckImportPolicyMarker(destination, markerOwner, markerStorePtr); + CheckImportPolicyOutput(destination, mode); + CHECK(destination.getAllDataPaths().size() == 4); + } + CheckSelectedImportDefaultResolver(incomingResolver, k_SelectedFormat.str()); +} + +namespace +{ +/** + * @struct SelectedImportTestContext + * @brief Keeps resident fixture setup and cache cleanup inside the saved resolver scope. + */ +struct SelectedImportTestContext +{ + std::shared_ptr memoryResolver = std::make_shared(); + // The guard also restores the incoming owner if fixture construction throws. + decltype(UnitTest::SetDefaultFormatResolverForScope(nullptr)) RestoreResolver = UnitTest::SetDefaultFormatResolverForScope(memoryResolver); + ScopedTempFile file; + DREAM3D::Dream3dPreflightCache& cache = DREAM3D::Dream3dPreflightCache::Instance(); + + explicit SelectedImportTestContext(const std::string& name) + : file(fs::path(unit_test::k_BinaryTestOutputDir.view()) / name) + { + cache.clear(); + WriteImportPolicyFile(file.path); + } + + ~SelectedImportTestContext() noexcept + { + cache.clear(); + } + + SelectedImportTestContext(const SelectedImportTestContext&) = delete; + SelectedImportTestContext(SelectedImportTestContext&&) = delete; + SelectedImportTestContext& operator=(const SelectedImportTestContext&) = delete; + SelectedImportTestContext& operator=(SelectedImportTestContext&&) = delete; +}; + +std::shared_ptr CreateSelectedImportMarker(DataStructure& destination) +{ + auto* marker = CreateImportPolicyArray(destination, k_ImportPolicyMarker, k_ImportPolicyMarkerValues); + marker->getMetadata().setData("policy-marker", std::string("keep-existing-metadata")); + return destination.getSharedData(k_ImportPolicyMarker); +} + +class LaterImportPlanFailure : public CompleteImportContextResolver +{ +public: + LaterImportPlanFailure() = default; + ~LaterImportPlanFailure() override = default; + LaterImportPlanFailure(const LaterImportPlanFailure&) = default; + LaterImportPlanFailure(LaterImportPlanFailure&&) = default; + LaterImportPlanFailure& operator=(const LaterImportPlanFailure&) = default; + LaterImportPlanFailure& operator=(LaterImportPlanFailure&&) = default; + + bool allocation = false; + + std::string resolveFormat(const DataStructure& structure, const DataPath& path, DataType type, uint64 bytes) const override + { + auto format = CompleteImportContextResolver::resolveFormat(structure, path, type, bytes); + if(paths.size() == 2) + { + if(allocation) + { + throw std::bad_alloc(); + } + throw std::runtime_error("second selected plan witness"); + } + return format; + } +}; +} // namespace + +TEST_CASE("Selected import test context restores nested resolver ownership", "[StorageFormatPlan][SelectedImportPolicy][FormatResolver]") +{ + UnitTest::LoadPlugins(); + const auto registration = Application::GetOrCreateInstance()->getIOCollection().addIOManager(std::make_shared()); + SIMPLNX_RESULT_REQUIRE_VALID(registration); + const bool throwFromNestedScope = GENERATE(false, true); + CAPTURE(throwFromNestedScope); + auto incomingResolver = std::make_shared(k_SelectedFormat.str()); + const std::weak_ptr incomingLifetime = incomingResolver; + const auto restoreIncoming = UnitTest::SetDefaultFormatResolverForScope(std::move(incomingResolver)); + REQUIRE(incomingResolver == nullptr); + { + const SelectedImportTestContext outerContext("selected_import_outer_resolver.dream3d"); + REQUIRE_FALSE(incomingLifetime.expired()); + CheckSelectedImportDefaultResolver(outerContext.memoryResolver, ""); + bool caughtException = false; + try + { + const SelectedImportTestContext innerContext("selected_import_inner_resolver.dream3d"); + CheckSelectedImportDefaultResolver(innerContext.memoryResolver, ""); + if(throwFromNestedScope) + { + throw std::runtime_error("selected import context unwind"); + } + } catch(const std::runtime_error&) + { + caughtException = true; + } + CHECK(caughtException == throwFromNestedScope); + CheckSelectedImportDefaultResolver(outerContext.memoryResolver, ""); + } + REQUIRE_FALSE(incomingLifetime.expired()); + CheckSelectedImportDefaultResolver(incomingLifetime.lock(), k_SelectedFormat.str()); +} + +TEST_CASE("C8 later destination planning failure preserves destination and allocation identity", "[C8][SelectedImportPolicy]") +{ + SelectedImportTestContext context("selected_import_later_plan.dream3d"); + const bool allocation = GENERATE(false, true); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + CAPTURE(allocation, static_cast(mode)); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + const auto* markerStore = dynamic_cast(marker.get())->getIDataStore(); + const auto savedIds = destination.getAllDataObjectIds(); + const auto nextId = destination.getNextId(); + const auto resolver = std::make_shared(); + resolver->allocation = allocation; + destination.setFormatResolver(resolver); + const ImportH5ObjectPathsAction action(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst, k_ImportPolicySecond}); + if(allocation) + { + REQUIRE_THROWS_AS(action.apply(destination, mode), std::bad_alloc); + } + else + { + const auto result = action.apply(destination, mode); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().message.find("second selected plan witness") != std::string::npos); + } + CheckImportPolicyRecords(*resolver); + CHECK(resolver->completeContextCalls == 2); + CHECK(resolver->placeholderContextCalls == 2); + CHECK(destination.getAllDataObjectIds() == savedIds); + CHECK(destination.getAllDataPaths() == std::vector{k_ImportPolicyMarker}); + CHECK(destination.getNextId() == nextId); + CheckImportPolicyMarker(destination, marker, markerStore); +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +TEST_CASE("C8 publication faults remove exact objects and permit immediate ID reuse", "[C8][SelectedImportPublication]") +{ + UnitTest::LoadPlugins(); + using FaultType = ImportH5ObjectPathsAction::PublicationFaultType; + const auto fault = + GENERATE(FaultType::BeforePublication, FaultType::AfterSecondInsert, FaultType::AfterHierarchy, FaultType::BeforeTracking, FaultType::AllocationAfterSecond, FaultType::ReturnFalse); + const auto mode = GENERATE(IDataAction::Mode::Preflight, IDataAction::Mode::Execute); + CAPTURE(static_cast(fault), static_cast(mode)); + SelectedImportTestContext context("selected_import_publication_faults.dream3d"); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + const auto* markerStore = dynamic_cast(marker.get())->getIDataStore(); + destination.setFormatResolver(context.memoryResolver); + const auto savedIds = destination.getAllDataObjectIds(); + const auto savedNextId = destination.getNextId(); + std::vector removals; + usize additions = 0; + nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr& message) { + if(message->getMsgType() == DataRemovedMessage::MsgType) + { + removals.push_back(std::static_pointer_cast(message)->getId()); + } + else if(message->getMsgType() == DataAddedMessage::MsgType) + { + ++additions; + } + })); + ImportH5ObjectPathsAction action(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst, k_ImportPolicySecond}); + action.setPublicationFaultForTesting(fault); + if(fault == FaultType::AllocationAfterSecond) + { + REQUIRE_THROWS_AS(action.apply(destination, mode), std::bad_alloc); + } + else + { + const auto result = action.apply(destination, mode); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().code == -6214); + } + REQUIRE(action.publicationFaultConsumedForTesting()); + CHECK(destination.getAllDataObjectIds() == savedIds); + CHECK(destination.getAllDataPaths() == std::vector{k_ImportPolicyMarker}); + CHECK(destination.getNextId() == savedNextId); + CheckImportPolicyMarker(destination, marker, markerStore); + usize expectedRemovals = 1; + if(fault == FaultType::BeforePublication) + { + expectedRemovals = 0; + } + else if(fault == FaultType::AfterSecondInsert || fault == FaultType::AllocationAfterSecond) + { + expectedRemovals = 2; + } + CHECK(removals.size() == expectedRemovals); + if(expectedRemovals == 2) + { + CHECK((removals == std::vector{savedNextId + 1, savedNextId})); + } + else if(expectedRemovals == 1) + { + CHECK((removals == std::vector{savedNextId})); + } + CHECK(additions == 0); + auto retained = action.retainedOwnerForTesting(); + if(retained != nullptr) + { + CHECK(retained->getDataStructure() == nullptr); + } + const ImportH5ObjectPathsAction retry(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst, k_ImportPolicySecond}); + auto retried = retry.apply(destination, mode); + SIMPLNX_RESULT_REQUIRE_VALID(retried); + CheckImportPolicyOutput(destination, mode); + CHECK(additions == 0); + const auto nextId = destination.getNextId(); + auto* later = DataGroup::Create(destination, "AfterRetry"); + REQUIRE(later != nullptr); + CHECK(later->getId() == nextId); + CHECK(destination.getData(nextId) == later); + CHECK(destination.getData(DataPath({"AfterRetry"})) == later); + const auto beforeRelease = removals.size(); + retained.reset(); + action.setPublicationFaultForTesting(FaultType::None); + CHECK(removals.size() == beforeRelease); + CHECK(additions == 1); + REQUIRE(destination.removeData(nextId)); + CHECK(removals.size() == beforeRelease + 1); + CHECK(removals.back() == nextId); + const auto idsAfterOrdinaryRemoval = destination.getAllDataObjectIds(); + CHECK(std::find(idsAfterOrdinaryRemoval.begin(), idsAfterOrdinaryRemoval.end(), nextId) != idsAfterOrdinaryRemoval.end()); + UnitTest::CheckArraysInheritTupleDims(destination); +} + +TEST_CASE("C8 refused partial publication retains ownership and reserves identifiers", "[C8][SelectedImportPublication]") +{ + using FaultType = ImportH5ObjectPathsAction::PublicationFaultType; + const auto fault = GENERATE(FaultType::AfterHierarchy, FaultType::BeforeTracking, FaultType::AfterSecondInsert); + CAPTURE(static_cast(fault)); + SelectedImportTestContext context("selected_import_refused_cleanup.dream3d"); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + const auto* markerStore = dynamic_cast(marker.get())->getIDataStore(); + const auto savedNextId = destination.getNextId(); + ImportH5ObjectPathsAction action(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst, k_ImportPolicySecond}); + action.setPublicationFaultForTesting(fault, true); + const auto result = action.apply(destination, IDataAction::Mode::Execute); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() >= 2); + CHECK(result.errors().front().code == -6214); + CHECK(result.errors()[1].code == -6215); + REQUIRE(action.publicationFaultConsumedForTesting()); + REQUIRE(action.cleanupFaultConsumedForTesting()); + const auto retained = action.retainedOwnerForTesting(); + REQUIRE(retained != nullptr); + const DataPath retainedPath({k_ImportPolicyGroup.getTargetName(), retained->getName()}); + REQUIRE(destination.getData(retainedPath) == retained.get()); + REQUIRE(destination.getData(k_ImportPolicyGroup) != nullptr); + CHECK(destination.getData(retained->getId()) == (fault == FaultType::AfterSecondInsert ? retained.get() : nullptr)); + CHECK(destination.getNextId() > retained->getId()); + CHECK(destination.getNextId() > savedNextId); + auto* unrelated = DataGroup::Create(destination, "AfterRefusal"); + REQUIRE(unrelated != nullptr); + CHECK(unrelated->getId() > retained->getId()); + const auto advancedNextId = destination.getNextId(); + const auto cleaned = action.cleanupFailedImportForTesting(destination); + SIMPLNX_RESULT_REQUIRE_VALID(cleaned); + CHECK_FALSE(destination.containsData(k_ImportPolicyGroup)); + CHECK(destination.getData(retained->getId()) == nullptr); + const auto remainingIds = destination.getAllDataObjectIds(); + CHECK(std::find(remainingIds.begin(), remainingIds.end(), retained->getId()) == remainingIds.end()); + CHECK(retained->getDataStructure() == nullptr); + CHECK(destination.getData(DataPath({"AfterRefusal"})) == unrelated); + CHECK(destination.getNextId() == advancedNextId); + CheckImportPolicyMarker(destination, marker, markerStore); + const ImportH5ObjectPathsAction retry(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst, k_ImportPolicySecond}); + auto retried = retry.apply(destination, IDataAction::Mode::Execute); + SIMPLNX_RESULT_REQUIRE_VALID(retried); + CheckImportPolicyOutput(destination, IDataAction::Mode::Execute); +} + +TEST_CASE("C8 rollback settles all objects before callbacks and preserves callback IDs", "[C8][SelectedImportPublication]") +{ + const bool throwingObserver = GENERATE(false, true); + CAPTURE(throwingObserver); + SelectedImportTestContext context("selected_import_observer_cleanup.dream3d"); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + const auto savedNextId = destination.getNextId(); + usize removed = 0; + bool fullyDetachedAtEveryCallback = true; + DataObject::IdType callbackId = 0; + nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr& message) { + if(message->getMsgType() != DataRemovedMessage::MsgType) + { + return; + } + ++removed; + fullyDetachedAtEveryCallback = fullyDetachedAtEveryCallback && !destination.containsData(k_ImportPolicyGroup); + if(throwingObserver) + { + throw std::runtime_error("checked removal observer witness"); + } + if(callbackId == 0) + { + auto* added = DataGroup::Create(destination, "ObserverAdded"); + if(added != nullptr) + { + callbackId = added->getId(); + } + } + })); + const ImportH5ObjectPathsAction action(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst, k_ImportPolicySecond}); + action.setPublicationFaultForTesting(ImportH5ObjectPathsAction::PublicationFaultType::AfterSecondInsert); + const auto result = action.apply(destination, IDataAction::Mode::Execute); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().code == -6214); + CHECK(removed == 2); + CHECK(fullyDetachedAtEveryCallback); + CHECK_FALSE(destination.containsData(k_ImportPolicyGroup)); + if(throwingObserver) + { + REQUIRE(result.errors().size() == 3); + CHECK(result.errors()[1].code == -6216); + CHECK(result.errors()[2].code == -6216); + CHECK(destination.getNextId() == savedNextId); + } + else + { + REQUIRE(callbackId != 0); + CHECK(destination.getData(DataPath({"ObserverAdded"})) == destination.getData(callbackId)); + CHECK(destination.getNextId() == callbackId + 1); + CHECK(destination.getAllDataObjectIds().size() == 2); + } + CHECK(destination.getData(marker->getId()) == marker.get()); +} +#endif + +TEST_CASE("C8 import rejects expired ID collisions and checked ID overflow before publication", "[C8][SelectedImportPublication]") +{ + const bool overflow = GENERATE(false, true); + SelectedImportTestContext context("selected_import_id_precheck.dream3d"); + DataStructure destination; + auto* old = DataGroup::Create(destination, "Expired"); + REQUIRE(old != nullptr); + const auto expiredId = old->getId(); + REQUIRE(destination.removeData(expiredId)); + REQUIRE(destination.getData(expiredId) == nullptr); + const auto beforeIds = destination.getAllDataObjectIds(); + REQUIRE(std::find(beforeIds.begin(), beforeIds.end(), expiredId) != beforeIds.end()); + const auto requestedNext = overflow ? std::numeric_limits::max() - 1 : expiredId; + destination.setNextId(requestedNext); + const ImportH5ObjectPathsAction action(context.file.path, {k_ImportPolicyGroup, k_ImportPolicyFirst}); + const auto result = action.apply(destination, IDataAction::Mode::Preflight); + REQUIRE(result.invalid()); + REQUIRE_FALSE(result.errors().empty()); + CHECK(result.errors().front().code == -6213); + CHECK(destination.getAllDataObjectIds() == beforeIds); + CHECK(destination.getAllDataPaths().empty()); + CHECK(destination.getNextId() == requestedNext); + const ImportH5ObjectPathsAction empty(context.file.path, {}); + REQUIRE(empty.apply(destination, IDataAction::Mode::Preflight).valid()); + const ImportH5ObjectPathsAction absent(context.file.path, {DataPath({"Absent"})}); + REQUIRE(absent.apply(destination, IDataAction::Mode::Preflight).valid()); + CHECK(destination.getAllDataObjectIds() == beforeIds); + CHECK(destination.getNextId() == requestedNext); +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +TEST_CASE("C8 rollback preserves an existing destination parent and its other children", "[C8][SelectedImportPublication]") +{ + SelectedImportTestContext context("selected_import_existing_parent_rollback.dream3d"); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + auto* parent = DataGroup::Create(destination, k_ImportPolicyGroup.getTargetName()); + REQUIRE(parent != nullptr); + const DataPath otherPath({"SelectedGroup", "Other"}); + auto* other = CreateImportPolicyArray(destination, otherPath, k_ImportPolicyMarkerValues, parent->getId()); + const auto parentId = parent->getId(); + const auto otherId = other->getId(); + const auto nextId = destination.getNextId(); + const auto ids = destination.getAllDataObjectIds(); + const ImportH5ObjectPathsAction action(context.file.path, {k_ImportPolicyFirst, k_ImportPolicySecond}); + action.setPublicationFaultForTesting(ImportH5ObjectPathsAction::PublicationFaultType::AfterHierarchy); + const auto failed = action.apply(destination, IDataAction::Mode::Execute); + REQUIRE(failed.invalid()); + REQUIRE(action.publicationFaultConsumedForTesting()); + CHECK(destination.getData(parentId) == parent); + CHECK(destination.getData(k_ImportPolicyGroup) == parent); + CHECK(destination.getData(otherId) == other); + CHECK(parent->getSize() == 1); + CHECK(destination.getAllDataObjectIds() == ids); + CHECK(destination.getNextId() == nextId); + CheckImportPolicyArray(destination, otherPath, k_ImportPolicyMarkerValues, IDataAction::Mode::Execute); + CHECK(destination.getData(marker->getId()) == marker.get()); + UnitTest::CheckArraysInheritTupleDims(destination); +} +#endif + +#if !defined(SIMPLNX_STORE_COPY_STRICT_FORMAT) || !SIMPLNX_STORE_COPY_STRICT_FORMAT +namespace +{ +const DataPath k_C8FinalizerLists({"SelectedGroup", "Lists"}); +const DataPath k_C8FinalizerStrings({"SelectedGroup", "Strings"}); +constexpr int32 k_C8FinalizerWarning = -89531; +constexpr int32 k_C8SecondLeafFailure = -89532; +constexpr int32 k_C8DuplicateFinalizer = -89533; +const DataPath k_C8LegacyContainer({"PlanContainer"}); +const DataPath k_C8LegacyMatrix({"PlanContainer", "CellData"}); +const DataPath k_C8LegacyFirst({"PlanContainer", "CellData", "First"}); +const DataPath k_C8LegacySecond({"PlanContainer", "CellData", "Second"}); + +/** + * @class C8SingleImportFinalizer + * @brief Observes the actual finalizer callback and materializes each tiny selected leaf once. + * + * The in-core tests register this manager only when no finalizer is active. The manager is disabled before its test ends. + */ +class C8SingleImportFinalizer : public IDataIOManager +{ +public: + C8SingleImportFinalizer() = default; + ~C8SingleImportFinalizer() override = default; + C8SingleImportFinalizer(const C8SingleImportFinalizer&) = default; + C8SingleImportFinalizer(C8SingleImportFinalizer&&) = default; + C8SingleImportFinalizer& operator=(const C8SingleImportFinalizer&) = default; + C8SingleImportFinalizer& operator=(C8SingleImportFinalizer&&) = default; + + bool active = false; + bool failOnSecondLeaf = false; + bool failureConsumed = false; + usize calls = 0; + usize leafAttempts = 0; + usize numericReads = 0; + bool completeEmptyContext = false; + std::vector expectedLeaves; + std::vector numericPaths; + std::vector ancestors; + std::vector observedLeaves; + std::vector completedLeaves; + + std::string formatName() const override + { + return "C8-single-import-finalizer"; + } + + bool finalizesImport() const override + { + return active; + } + + /** + * @brief Checks staged plans and completes each selected leaf through existing readers. + * @param structure Supplies the prepared destination context. + * @param paths Supplies only the selected numeric, list, and string leaves. + * @param reader Supplies the closed-and-reopened source fixture. + * @return Completed-read warning or the deliberate second-leaf or duplicate-call error. + */ + Result<> onImportFinalize(DataStructure& structure, const std::vector& paths, const HDF5::FileIO& reader) override + { + ++calls; + if(calls != 1) + { + return MakeErrorResult(k_C8DuplicateFinalizer, "duplicate selected-import finalizer witness"); + } + observedLeaves = paths; + auto actual = paths; + auto expected = expectedLeaves; + std::sort(actual.begin(), actual.end()); + std::sort(expected.begin(), expected.end()); + completeEmptyContext = actual == expected && structure.containsData(k_ImportPolicyMarker) && !structure.containsData(k_ImportPolicyExcluded); + for(const auto& path : ancestors) + { + completeEmptyContext = completeEmptyContext && structure.getDataAs(path) != nullptr; + } + for(const auto& path : numericPaths) + { + const auto* array = structure.getDataAs(path); + completeEmptyContext = completeEmptyContext && array != nullptr && array->getStoreType() == IDataStore::StoreType::Empty && array->getDataFormat().empty() && + array->getIDataStore()->getPlannedStoreType() == IDataStore::StoreType::InMemory; + } + if(std::find(paths.begin(), paths.end(), k_C8FinalizerLists) != paths.end()) + { + const auto* lists = structure.getDataAs>(k_C8FinalizerLists); + const auto* strings = structure.getDataAs(k_C8FinalizerStrings); + completeEmptyContext = completeEmptyContext && lists != nullptr && dynamic_cast*>(lists->getStore().get()) != nullptr && strings != nullptr && strings->isPlaceholder(); + } + if(!completeEmptyContext) + { + return MakeErrorResult(-89534, "selected-import finalizer received incomplete or materialized metadata"); + } + + Result<> result; + const std::string root = DREAM3D::GetFileVersion(reader) == DREAM3D::k_LegacyFileVersion ? "/DataContainers/" : "/DataStructure/"; + for(const auto& path : expectedLeaves) + { + ++leafAttempts; + if(failOnSecondLeaf && leafAttempts == 2) + { + failureConsumed = true; + auto failure = MakeErrorResult(k_C8SecondLeafFailure, "second selected leaf materialization witness"); + failure.warnings() = std::move(result.warnings()); + return failure; + } + Result<> step; + if(auto* array = structure.getDataAs(path); array != nullptr) + { + const auto dataset = reader.openDataset(root + path.toString()); + ++numericReads; + auto read = HDF5::DataStoreIO::ReadDataStoreIntoMemory(dataset); + if(read.invalid()) + { + return MergeResults(std::move(result), ConvertResult(std::move(read))); + } + if(read.value() == nullptr) + { + return MergeResults(std::move(result), MakeErrorResult(-89535, "finalizer fixture unexpectedly returned a recovery placeholder")); + } + auto warnings = std::move(read.warnings()); + step = array->setDataStore(std::move(read.value())); + step.warnings().insert(step.warnings().begin(), std::make_move_iterator(warnings.begin()), std::make_move_iterator(warnings.end())); + } + else + { + step = HDF5::DataStructureReader::FinishImportingObject(structure, reader, path); + } + result = MergeResults(std::move(result), std::move(step)); + if(result.invalid()) + { + return result; + } + completedLeaves.push_back(path); + if(completedLeaves.size() == 1) + { + result.warnings().push_back({k_C8FinalizerWarning, "first selected leaf materialized"}); + } + } + return result; + } +}; + +/** + * @brief Writes a mixed NX fixture and validates its numeric values before the finalizer is active. + * @param path Identifies the closed source file. + */ +void WriteC8FinalizerFile(const fs::path& path) +{ + DataStructure source; + source.setFormatResolver(std::make_shared()); + auto* group = DataGroup::Create(source, k_ImportPolicyGroup.getTargetName()); + REQUIRE(group != nullptr); + CreateImportPolicyArray(source, k_ImportPolicyFirst, k_ImportPolicyFirstValues, group->getId()); + CreateImportPolicyArray(source, k_ImportPolicySecond, k_ImportPolicySecondValues, group->getId()); + CreateImportPolicyArray(source, k_ImportPolicyExcluded, k_ImportPolicyExcludedValues, group->getId()); + auto* lists = NeighborList::Create(source, "Lists", ShapeType{3}, group->getId()); + REQUIRE(lists != nullptr); + lists->setList(0, std::vector{31, -7}); + lists->setList(1, std::vector{}); + lists->setList(2, std::vector{9}); + REQUIRE(StringArray::CreateWithValues(source, "Strings", ShapeType{2}, {"first", "last"}, group->getId()) != nullptr); + auto write = DREAM3D::WriteFile(path, source); + SIMPLNX_RESULT_REQUIRE_VALID(write); + { + const auto reader = HDF5::FileIO::ReadFile(path); + REQUIRE(reader.isValid()); + CheckImportPolicyFileArray(reader, k_ImportPolicyFirst, k_ImportPolicyFirstValues); + CheckImportPolicyFileArray(reader, k_ImportPolicySecond, k_ImportPolicySecondValues); + CheckImportPolicyFileArray(reader, k_C8FinalizerLists, std::array{31, -7, 9}); + CheckImportPolicyFileArray(reader, k_ImportPolicyGroup.createChildPath(lists->getNumNeighborsArrayName()), std::array{2, 0, 1}); + } + fs::last_write_time(path, fs::file_time_type::clock::now() - std::chrono::seconds(10)); +} + +/** + * @brief Creates two bounded legacy arrays and one real unsupported-object warning. + * @param path Identifies the closed fixture file. + */ +void WriteC8LegacyPlanWarningFile(const fs::path& path) +{ + { + auto file = HDF5::FileIO::WriteFile(path); + REQUIRE(file.isValid()); + REQUIRE(file.writeStringAttribute("FileVersion", DREAM3D::k_LegacyFileVersion.str()).valid()); + auto containers = file.createGroup("DataContainers"); + auto container = containers.createGroup("PlanContainer"); + auto matrix = container.createGroup("CellData"); + REQUIRE(matrix.writeVectorAttribute("TupleDimensions", ShapeType{3}).valid()); + REQUIRE(matrix.writeScalarAttribute("AttributeMatrixType", 13).valid()); + for(const std::string name : {"First", "Second"}) + { + auto dataset = matrix.createDataset(name); + const std::array values = name == "First" ? std::array{11, -4, 6} : std::array{27, 0, -19}; + REQUIRE(dataset.writeSpan({3, 1}, nonstd::span(values.data(), values.size())).valid()); + REQUIRE(dataset.writeStringAttribute(Constants::k_ObjectTypeTag, Int32Array::GetTypeName()).valid()); + REQUIRE(dataset.writeVectorAttribute("TupleDimensions", ShapeType{3}).valid()); + REQUIRE(dataset.writeVectorAttribute("ComponentDimensions", ShapeType{1}).valid()); + } + auto unsupported = matrix.createGroup("WarningSource"); + REQUIRE(unsupported.writeStringAttribute(Constants::k_ObjectTypeTag, "UnsupportedLegacyObject").valid()); + } + { + const auto reader = HDF5::FileIO::ReadFile(path); + REQUIRE(reader.isValid()); + const auto fileId = reader.getId(); + const std::lock_guard lock(HDF5::Support::ApiLock()); + for(const std::string name : {"First", "Second"}) + { + const std::string datasetPath = "/DataContainers/PlanContainer/CellData/" + name; + const hid_t dataset = H5Dopen2(fileId, datasetPath.c_str(), H5P_DEFAULT); + REQUIRE(dataset >= 0); + const auto datasetGuard = MakeScopeGuard([dataset]() noexcept { H5Dclose(dataset); }); + const hid_t space = H5Dget_space(dataset); + REQUIRE(space >= 0); + const auto spaceGuard = MakeScopeGuard([space]() noexcept { H5Sclose(space); }); + REQUIRE(H5Sget_simple_extent_ndims(space) == 2); + REQUIRE(H5Sget_simple_extent_npoints(space) == 3); + std::array shape{}; + REQUIRE(H5Sget_simple_extent_dims(space, shape.data(), nullptr) == 2); + REQUIRE(shape == std::array{3, 1}); + std::array values{}; + REQUIRE(H5Dread(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, values.data()) >= 0); + const std::array expected = name == "First" ? std::array{11, -4, 6} : std::array{27, 0, -19}; + REQUIRE(values == expected); + } + } + fs::last_write_time(path, fs::file_time_type::clock::now() - std::chrono::seconds(10)); +} + +/** + * @class C8WarningPlanResolver + * @brief Records complete legacy plans and can reject the second plan after disk warnings exist. + */ +class C8WarningPlanResolver : public RecordingFormatResolver +{ +public: + C8WarningPlanResolver() = default; + ~C8WarningPlanResolver() override = default; + C8WarningPlanResolver(const C8WarningPlanResolver&) = default; + C8WarningPlanResolver(C8WarningPlanResolver&&) = default; + C8WarningPlanResolver& operator=(const C8WarningPlanResolver&) = default; + C8WarningPlanResolver& operator=(C8WarningPlanResolver&&) = default; + + bool failOnSecond = false; + mutable usize completeEmptyCalls = 0; + + std::string resolveFormat(const DataStructure& structure, const DataPath& path, DataType type, uint64 bytes) const override + { + const auto* first = structure.getDataAs(k_C8LegacyFirst); + const auto* second = structure.getDataAs(k_C8LegacySecond); + if(first != nullptr && second != nullptr && first->getStoreType() == IDataStore::StoreType::Empty && second->getStoreType() == IDataStore::StoreType::Empty && + structure.containsData(k_ImportPolicyMarker)) + { + ++completeEmptyCalls; + } + auto format = RecordingFormatResolver::resolveFormat(structure, path, type, bytes); + if(failOnSecond && paths.size() == 2) + { + throw std::runtime_error("second selected plan after disk warning witness"); + } + return format; + } +}; +} // namespace + +TEST_CASE("C8 single active finalizer observes complete plans and exact leaves once", "[C8][SelectedImportFinalizer]") +{ + const auto application = Application::GetOrCreateInstance(); + SelectedImportTestContext context("c8_single_finalizer.dream3d"); + WriteC8FinalizerFile(context.file.path); + auto& collection = application->getIOCollection(); + REQUIRE_FALSE(collection.anyManagerFinalizesImport()); + const auto manager = std::make_shared(); + const auto disable = MakeScopeGuard([manager]() noexcept { manager->active = false; }); + auto registration = collection.addIOManager(manager); + SIMPLNX_RESULT_REQUIRE_VALID(registration); + manager->expectedLeaves = {k_ImportPolicyFirst, k_ImportPolicySecond, k_C8FinalizerLists, k_C8FinalizerStrings}; + manager->numericPaths = {k_ImportPolicyFirst, k_ImportPolicySecond}; + manager->ancestors = {k_ImportPolicyGroup}; + manager->active = true; + REQUIRE(collection.anyManagerFinalizesImport()); + REQUIRE(collection.getManager(manager->formatName()) == manager); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + const auto* markerStore = dynamic_cast(marker.get())->getIDataStore(); + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const ImportH5ObjectPathsAction action(context.file.path, {k_C8FinalizerStrings, k_ImportPolicySecond, k_ImportPolicyGroup, k_C8FinalizerLists, k_ImportPolicyFirst}); + auto result = action.apply(destination, IDataAction::Mode::Execute); + SIMPLNX_RESULT_REQUIRE_VALID(result); + CHECK(manager->calls == 1); + CHECK(manager->completeEmptyContext); + CHECK(manager->leafAttempts == 4); + CHECK(manager->numericReads == 2); + CHECK(manager->completedLeaves == manager->expectedLeaves); + REQUIRE(result.warnings().size() == 1); + CHECK(result.warnings()[0].code == k_C8FinalizerWarning); + CheckImportPolicyRecords(*resolver); + CHECK(resolver->completeContextCalls == 2); + CheckImportPolicyOutput(destination, IDataAction::Mode::Execute); + CheckImportPolicyMarker(destination, marker, markerStore); + const auto* lists = destination.getDataAs>(k_C8FinalizerLists); + const auto* strings = destination.getDataAs(k_C8FinalizerStrings); + REQUIRE(lists != nullptr); + REQUIRE(strings != nullptr); + CHECK(lists->getList(0) == std::vector{31, -7}); + CHECK(lists->getList(1).empty()); + CHECK(lists->getList(2) == std::vector{9}); + CHECK(strings->values() == std::vector{"first", "last"}); + CHECK(destination.getAllDataPaths().size() == 6); + const auto idsBeforeDuplicate = destination.getAllDataObjectIds(); + const auto reader = HDF5::FileIO::ReadFile(context.file.path); + auto duplicate = manager->onImportFinalize(destination, manager->expectedLeaves, reader); + REQUIRE(duplicate.invalid()); + REQUIRE(duplicate.errors().size() == 1); + CHECK(duplicate.errors()[0].code == k_C8DuplicateFinalizer); + CHECK(manager->calls == 2); + CHECK(manager->numericReads == 2); + CHECK(manager->completedLeaves.size() == 4); + CHECK(destination.getAllDataObjectIds() == idsBeforeDuplicate); + UnitTest::CheckArraysInheritTupleDims(destination); +} + +#if defined(SIMPLNX_BUILD_TESTS) && SIMPLNX_BUILD_TESTS +TEST_CASE("C8 publication handoff preserves warning order and exact destination state", "[C8][SelectedImportPublication][SelectedImportFinalizer]") +{ + UnitTest::LoadPlugins(); + static_assert(noexcept(std::declval().swap(std::declval()))); + static_assert(std::is_nothrow_move_constructible_v); + enum class Outcome : uint8 + { + OrdinarySuccess, + InstrumentedSuccess, + PublicationFailure, + AllocationFailure + }; + const auto outcome = GENERATE(Outcome::OrdinarySuccess, Outcome::InstrumentedSuccess, Outcome::PublicationFailure, Outcome::AllocationFailure); + CAPTURE(static_cast(outcome)); + const auto application = Application::GetOrCreateInstance(); + const SelectedImportTestContext context("c8_publication_warning_handoff.dream3d"); + WriteC8LegacyPlanWarningFile(context.file.path); + auto& collection = application->getIOCollection(); + REQUIRE_FALSE(collection.anyManagerFinalizesImport()); + const auto manager = std::make_shared(); + const auto disable = MakeScopeGuard([manager]() noexcept { manager->active = false; }); + auto registration = collection.addIOManager(manager); + SIMPLNX_RESULT_REQUIRE_VALID(registration); + manager->expectedLeaves = {k_C8LegacyFirst, k_C8LegacySecond}; + manager->numericPaths = manager->expectedLeaves; + manager->ancestors = {k_C8LegacyContainer, k_C8LegacyMatrix}; + manager->active = true; + DataStructure destination; + const auto markerOwner = CreateSelectedImportMarker(destination); + const auto* markerStorePtr = dynamic_cast(markerOwner.get())->getIDataStore(); + REQUIRE(markerOwner->getId() == 1); + REQUIRE(destination.getNextId() == 2); + const auto resolver = std::make_shared(); + destination.setFormatResolver(resolver); + const ImportH5ObjectPathsAction action(context.file.path, {k_C8LegacySecond, k_C8LegacyContainer, k_C8LegacyFirst, k_C8LegacyMatrix}); + using FaultType = ImportH5ObjectPathsAction::PublicationFaultType; + if(outcome != Outcome::OrdinarySuccess) + { + auto fault = FaultType::None; + if(outcome == Outcome::AllocationFailure) + { + fault = FaultType::AllocationAfterSecond; + } + else if(outcome == Outcome::PublicationFailure) + { + fault = FaultType::AfterSecondInsert; + } + action.setPublicationFaultForTesting(fault); + } + std::vector removals; + usize additions = 0; + const nod::scoped_connection connection(destination.getSignal().connect([&](DataStructure*, const std::shared_ptr& message) { + if(message->getMsgType() == DataRemovedMessage::MsgType) + { + removals.push_back(std::static_pointer_cast(message)->getId()); + } + else if(message->getMsgType() == DataAddedMessage::MsgType) + { + ++additions; + } + })); + if(outcome == Outcome::AllocationFailure) + { + REQUIRE_THROWS_AS(action.apply(destination, IDataAction::Mode::Execute), std::bad_alloc); + } + else + { + const auto result = action.apply(destination, IDataAction::Mode::Execute); + if(outcome == Outcome::PublicationFailure) + { + SIMPLNX_RESULT_REQUIRE_INVALID(result); + REQUIRE(result.errors().size() == 1); + CHECK(result.errors()[0].code == -6214); + } + else + { + SIMPLNX_RESULT_REQUIRE_VALID(result); + } + REQUIRE(result.warnings().size() == 2); + CHECK(result.warnings()[0].code == -298012); + CHECK(result.warnings()[0].message.find("WarningSource") != std::string::npos); + CHECK(result.warnings()[1].code == k_C8FinalizerWarning); + CHECK(result.warnings()[1].message == "first selected leaf materialized"); + } + CHECK(manager->calls == 1); + CHECK(manager->completeEmptyContext); + CHECK(manager->leafAttempts == 2); + CHECK(manager->numericReads == 2); + CHECK(manager->completedLeaves == std::vector{k_C8LegacyFirst, k_C8LegacySecond}); + CHECK(resolver->completeEmptyCalls == 2); + CHECK(additions == 0); + CheckImportPolicyMarker(destination, markerOwner, markerStorePtr); + if(outcome == Outcome::PublicationFailure || outcome == Outcome::AllocationFailure) + { + REQUIRE(action.publicationFaultConsumedForTesting()); + CHECK((destination.getAllDataObjectIds() == std::vector{1})); + CHECK(destination.getAllDataPaths() == std::vector{k_ImportPolicyMarker}); + CHECK(destination.getNextId() == 2); + CHECK((removals == std::vector{3, 2})); + const auto retained = action.retainedOwnerForTesting(); + REQUIRE(retained != nullptr); + CHECK(retained->getId() == 3); + CHECK(retained->getDataStructure() == nullptr); + CHECK(retained->getParentIds().empty()); + } + else + { + CHECK((destination.getAllDataObjectIds() == std::vector{1, 2, 3, 4, 5})); + CHECK(destination.getNextId() == 6); + const auto* containerPtr = destination.getDataAs(k_C8LegacyContainer); + const auto* matrixPtr = destination.getDataAs(k_C8LegacyMatrix); + const auto* firstPtr = destination.getDataAs(k_C8LegacyFirst); + const auto* secondPtr = destination.getDataAs(k_C8LegacySecond); + REQUIRE(containerPtr != nullptr); + REQUIRE(matrixPtr != nullptr); + REQUIRE(firstPtr != nullptr); + REQUIRE(secondPtr != nullptr); + CHECK(destination.getData(DataObject::IdType{2}) == containerPtr); + CHECK(destination.getData(DataObject::IdType{3}) == matrixPtr); + CHECK(destination.getData(DataObject::IdType{4}) == firstPtr); + CHECK(destination.getData(DataObject::IdType{5}) == secondPtr); + CheckImportPolicyArray(destination, k_C8LegacyFirst, std::array{11, -4, 6}, IDataAction::Mode::Execute); + CheckImportPolicyArray(destination, k_C8LegacySecond, std::array{27, 0, -19}, IDataAction::Mode::Execute); + CHECK(removals.empty()); + CHECK_FALSE(action.publicationFaultConsumedForTesting()); + } + UnitTest::CheckArraysInheritTupleDims(destination); +} +#endif + +TEST_CASE("C8 planning and second leaf failures retain earlier warnings without publication", "[C8][SelectedImportFinalizer]") +{ + const bool planningFailure = GENERATE(false, true); + const bool warm = GENERATE(false, true); + CAPTURE(planningFailure, warm); + const auto application = Application::GetOrCreateInstance(); + SelectedImportTestContext context("c8_failure_warning_order.dream3d"); + WriteC8LegacyPlanWarningFile(context.file.path); + auto& collection = application->getIOCollection(); + REQUIRE_FALSE(collection.anyManagerFinalizesImport()); + const auto manager = std::make_shared(); + const auto disable = MakeScopeGuard([manager]() noexcept { manager->active = false; }); + auto registration = collection.addIOManager(manager); + SIMPLNX_RESULT_REQUIRE_VALID(registration); + manager->expectedLeaves = {k_C8LegacyFirst, k_C8LegacySecond}; + manager->numericPaths = manager->expectedLeaves; + manager->ancestors = {k_C8LegacyContainer, k_C8LegacyMatrix}; + manager->failOnSecondLeaf = true; + manager->active = true; + if(warm) + { + auto metadata = context.cache.fetchNeutralMetadata(context.file.path); + SIMPLNX_RESULT_REQUIRE_VALID(metadata); + REQUIRE(metadata.warnings().size() == 1); + REQUIRE(metadata.warnings()[0].code == -298012); + } + context.cache.resetStats(); + DataStructure destination; + const auto marker = CreateSelectedImportMarker(destination); + const auto* markerStore = dynamic_cast(marker.get())->getIDataStore(); + const auto beforeIds = destination.getAllDataObjectIds(); + const auto beforeNextId = destination.getNextId(); + const auto resolver = std::make_shared(); + resolver->failOnSecond = planningFailure; + destination.setFormatResolver(resolver); + const ImportH5ObjectPathsAction action(context.file.path, {k_C8LegacySecond, k_C8LegacyContainer, k_C8LegacyFirst, k_C8LegacyMatrix}); + auto result = action.apply(destination, IDataAction::Mode::Execute); + REQUIRE(result.invalid()); + REQUIRE(result.errors().size() == 1); + REQUIRE(result.warnings().size() == (planningFailure ? 1 : 2)); + CHECK(result.warnings()[0].code == -298012); + CHECK(result.warnings()[0].message.find("WarningSource") != std::string::npos); + CHECK(resolver->paths.size() == 2); + CHECK(std::count(resolver->paths.begin(), resolver->paths.end(), k_C8LegacyFirst) == 1); + CHECK(std::count(resolver->paths.begin(), resolver->paths.end(), k_C8LegacySecond) == 1); + CHECK(resolver->numericTypes == std::vector{DataType::int32, DataType::int32}); + CHECK(resolver->logicalBytes == std::vector{12, 12}); + CHECK(resolver->completeEmptyCalls == 2); + CHECK(context.cache.hitCount() == (warm ? 1 : 0)); + CHECK(context.cache.missCount() == (warm ? 0 : 1)); + if(planningFailure) + { + CHECK(result.errors()[0].code == -10601); + CHECK(result.errors()[0].message.find("second selected plan after disk warning witness") != std::string::npos); + CHECK(manager->calls == 0); + CHECK(manager->leafAttempts == 0); + } + else + { + CHECK(result.errors()[0].code == k_C8SecondLeafFailure); + CHECK(result.errors()[0].message == "second selected leaf materialization witness"); + CHECK(result.warnings()[1].code == k_C8FinalizerWarning); + CHECK(result.warnings()[1].message == "first selected leaf materialized"); + CHECK(manager->calls == 1); + CHECK(manager->completeEmptyContext); + CHECK(manager->leafAttempts == 2); + CHECK(manager->numericReads == 1); + CHECK(manager->completedLeaves == std::vector{k_C8LegacyFirst}); + CHECK(manager->failureConsumed); + } + CHECK(destination.getAllDataPaths() == std::vector{k_ImportPolicyMarker}); + CHECK(destination.getAllDataObjectIds() == beforeIds); + CHECK(destination.getNextId() == beforeNextId); + CHECK(&destination.formatResolver() == resolver.get()); + CheckImportPolicyMarker(destination, marker, markerStore); +} +#endif diff --git a/test/TestProfileSupport.cmake b/test/TestProfileSupport.cmake new file mode 100644 index 0000000000..3b0fbc7fb4 --- /dev/null +++ b/test/TestProfileSupport.cmake @@ -0,0 +1,8 @@ +# This include also serves test targets configured without SIMPLNX_BUILD_TESTS. +if(NOT TARGET simplnx::TestProfile) + add_library(SimplnxTestProfile INTERFACE) + add_library(simplnx::TestProfile ALIAS SimplnxTestProfile) + target_include_directories(SimplnxTestProfile INTERFACE + "${CMAKE_CURRENT_LIST_DIR}/UnitTestCommon/include") + target_compile_features(SimplnxTestProfile INTERFACE cxx_std_20) +endif() diff --git a/test/UnitTestCommon/CMakeLists.txt b/test/UnitTestCommon/CMakeLists.txt index 04ba954feb..0780ace17d 100644 --- a/test/UnitTestCommon/CMakeLists.txt +++ b/test/UnitTestCommon/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(Catch2 CONFIG REQUIRED) +include("${CMAKE_CURRENT_LIST_DIR}/../TestProfileSupport.cmake") add_library(SimplnxUnitTestCommon INTERFACE) add_library(simplnx::UnitTestCommon ALIAS SimplnxUnitTestCommon) @@ -13,6 +14,7 @@ target_link_libraries(SimplnxUnitTestCommon simplnx Catch2::Catch2 ZLIB::ZLIB + simplnx::TestProfile ) target_sources(SimplnxUnitTestCommon @@ -25,4 +27,5 @@ target_sources(SimplnxUnitTestCommon ${PROJECT_SOURCE_DIR}/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp ${PROJECT_SOURCE_DIR}/test/UnitTestCommon/include/simplnx/UnitTest/DataObjectComparison.hpp ${PROJECT_SOURCE_DIR}/test/UnitTestCommon/include/simplnx/UnitTest/HDF5DatasetProbe.hpp + ${PROJECT_SOURCE_DIR}/test/UnitTestCommon/include/simplnx/UnitTest/ScopedDefaultFormatResolver.hpp ) diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/AlgorithmTestScope.cpp b/test/UnitTestCommon/include/simplnx/UnitTest/AlgorithmTestScope.cpp index b3a7644e85..ef4d1ed9de 100644 --- a/test/UnitTestCommon/include/simplnx/UnitTest/AlgorithmTestScope.cpp +++ b/test/UnitTestCommon/include/simplnx/UnitTest/AlgorithmTestScope.cpp @@ -66,7 +66,7 @@ AlgorithmTestScope::AlgorithmTestScope(AlgorithmTestScenario scenario) : m_Scenario(scenario) { auto* preferences = Application::GetOrCreateInstance()->getPreferences(); - const int64 originalLargeDataSize = preferences->valueAs(Preferences::k_LargeDataSize_Key); + const nlohmann::json originalLargeDataSize = preferences->value(Preferences::k_LargeDataSize_Key); m_PreferencesSentinel = std::make_unique(DataStorageMode::ForceInCore, originalLargeDataSize); } diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/CatchSessionRunner.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/CatchSessionRunner.hpp new file mode 100644 index 0000000000..2479d7caed --- /dev/null +++ b/test/UnitTestCommon/include/simplnx/UnitTest/CatchSessionRunner.hpp @@ -0,0 +1,85 @@ +#pragma once + +#ifndef CATCH_CONFIG_RUNNER +#error "CatchSessionRunner.hpp requires a CATCH_CONFIG_RUNNER translation unit" +#endif + +#include + +#include +#include +#include +#include + +namespace nx::core::UnitTest +{ +/** + * @struct CatchSessionResult + * @brief Keeps successful listing counts distinct from Catch failures with the same integer status. + */ +struct CatchSessionResult +{ + int exitCode = 1; + bool successful = false; +}; + +/** + * @brief Runs Catch and destroys its Session before returning the outcome. + * @param argc Original command-line argument count. + * @param argv Original command-line arguments. + * @return Catch status and success, or an unsuccessful status of 255 after a caught exception. + * @pre No Catch Session has been constructed in this process. + * @note The caller owns profile installation and Application teardown. + */ +inline CatchSessionResult RunCatchSession(int argc, char* argv[]) +{ + CatchSessionResult result; + try + { + Catch::Session session; + result.exitCode = session.applyCommandLine(argc, argv); + if(result.exitCode == 0) + { + const auto& data = session.configData(); + const bool listing = data.listTests || data.listTestNamesOnly || data.listTags || data.listReporters; + if(listing && !data.showHelp && !data.libIdentify) + { + // Catch::Session maps listing exceptions and a successful count of 255 to the same status. + // Use Catch's listing API inside this exception boundary to distinguish those outcomes. + if((data.waitForKeypress & Catch::WaitForKeypress::BeforeStart) != 0) + { + Catch::cout() << "...waiting for enter/ return before starting" << std::endl; + static_cast(std::getchar()); + } + auto config = std::make_shared(data); + Catch::seedRng(*config); + if(data.filenamesAsTags) + { + Catch::applyFilenamesAsTags(*config); + } + result.exitCode = static_cast(*Catch::list(config)); + if((data.waitForKeypress & Catch::WaitForKeypress::BeforeExit) != 0) + { + Catch::cout() << "...waiting for enter/ return before exiting, with code: " << result.exitCode << std::endl; + static_cast(std::getchar()); + } + result.successful = true; + } + else + { + result.exitCode = session.run(); + result.successful = result.exitCode == 0; + } + } + } catch(const std::exception& exception) + { + std::fprintf(stderr, "Test runner failed: %s\n", exception.what()); + result = {255, false}; + } catch(...) + { + std::fputs("Test runner failed with an unknown exception\n", stderr); + result = {255, false}; + } + return result; +} +} // namespace nx::core::UnitTest diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/CodecOwnershipCountersForTesting.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/CodecOwnershipCountersForTesting.hpp new file mode 100644 index 0000000000..9b90e2237c --- /dev/null +++ b/test/UnitTestCommon/include/simplnx/UnitTest/CodecOwnershipCountersForTesting.hpp @@ -0,0 +1,81 @@ +#pragma once + +#include "simplnx/Utilities/Parsing/HDF5/ParallelChunkCodec.hpp" + +#include +#include +#include + +#if SIMPLNX_BUILD_TESTS +namespace nx::core::HDF5 +{ +/** + * @struct CodecOwnershipCountersForTesting + * @brief Collects concurrent payload lifetimes for one isolated observation scope. + */ +struct CodecOwnershipCountersForTesting +{ + std::atomic live{0}; + std::atomic peak{0}; + std::atomic nominalAcquired{0}; + std::atomic edgeAcquired{0}; + std::atomic transfers{0}; + std::atomic invalidRelease{false}; + + /** + * @brief Records one event without allocation, locks or storage reentry. + * @param context Points to the live fixed counter record. + * @param file Borrows the event identity; this isolated scope observes every event. + * @param dataset Borrows the dataset identity. + * @param event Identifies acquisition, destruction or transfer. + * @param bytes Supplies the actual observed capacity. + * @param category Identifies the legacy payload kind when applicable. + */ + static void Observe(void* context, const std::filesystem::path& file, std::string_view dataset, CodecIoEventForTesting event, uint64 bytes, uint64 category) noexcept + { + (void)file; + (void)dataset; + auto& self = *static_cast(context); + if(event == CodecIoEventForTesting::LegacyAcquired) + { + if(category == static_cast(LegacyPayloadKindForTesting::Nominal)) + { + self.nominalAcquired.fetch_add(1, std::memory_order_relaxed); + } + else if(category == static_cast(LegacyPayloadKindForTesting::Edge)) + { + self.edgeAcquired.fetch_add(1, std::memory_order_relaxed); + } + } + if(event == CodecIoEventForTesting::LegacyTransferred) + { + self.transfers.fetch_add(1, std::memory_order_relaxed); + } + if(event == CodecIoEventForTesting::RawAcquired || event == CodecIoEventForTesting::LegacyAcquired) + { + const uint64 current = self.live.fetch_add(bytes, std::memory_order_relaxed) + bytes; + uint64 peakValue = self.peak.load(std::memory_order_relaxed); + while(current > peakValue && !self.peak.compare_exchange_weak(peakValue, current, std::memory_order_relaxed)) + { + } + } + else if(event == CodecIoEventForTesting::RawReleased || event == CodecIoEventForTesting::LegacyReleased || event == CodecIoEventForTesting::LegacyTransferred) + { + uint64 current = self.live.load(std::memory_order_relaxed); + while(true) + { + if(bytes > current) + { + self.invalidRelease.store(true, std::memory_order_relaxed); + break; + } + if(self.live.compare_exchange_weak(current, current - bytes, std::memory_order_relaxed)) + { + break; + } + } + } + } +}; +} // namespace nx::core::HDF5 +#endif diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/OwnedCatchRunner.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/OwnedCatchRunner.hpp new file mode 100644 index 0000000000..99532d881d --- /dev/null +++ b/test/UnitTestCommon/include/simplnx/UnitTest/OwnedCatchRunner.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include "simplnx/Core/Application.hpp" +#include "simplnx/Core/Preferences.hpp" +#include "simplnx/UnitTest/CatchSessionRunner.hpp" +#include "simplnx/UnitTest/ProcessTestProfile.hpp" +#include "simplnx/UnitTest/TestApplicationCleanup.hpp" + +#include +#include +#include +#include + +namespace nx::core::UnitTest +{ +/** + * @brief Runs Catch with profile routing installed before listener construction. + * @param argc Original command-line argument count. + * @param argv Original command-line arguments. + * @return Catch status, or a nonzero setup or observable teardown failure. + */ +inline int RunOwnedCatchSession(int argc, char* argv[]) +{ + ProcessTestProfile profile; + std::string error; + CatchSessionResult result; + try + { + if(Application::Instance() != nullptr) + { + std::cerr << "Application exists before test profile installation\n"; + return 1; + } + if(!profile.install(error)) + { + std::cerr << "Test profile installation failed: " << error << '\n'; + return 1; + } + if(Preferences::DefaultFilePath("DREAM3DNX") != profile.corePreferencesFile()) + { + std::cerr << "Core preferences do not resolve to the installed test profile\n"; + } + else + { + result = RunCatchSession(argc, argv); + } + } catch(const std::exception& exception) + { + std::cerr << "Test runner failed: " << exception.what() << '\n'; + result = {255, false}; + } catch(...) + { + std::fputs("Test runner failed with an unknown exception\n", stderr); + result = {255, false}; + } + + // RunCatchSession destroys Session and its listeners before checked Application teardown. + if(!FinishTestApplication()) + { + result = {1, false}; + } + const int exitCode = profile.finishExitCode(result.exitCode, result.successful, error); + if(!error.empty()) + { + std::cerr << error << '\n'; + } + return exitCode; +} +} // namespace nx::core::UnitTest diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/ProcessTestProfile.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/ProcessTestProfile.hpp new file mode 100644 index 0000000000..7e6d2ffbc7 --- /dev/null +++ b/test/UnitTestCommon/include/simplnx/UnitTest/ProcessTestProfile.hpp @@ -0,0 +1,376 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#define SIMPLNX_TEST_RESTORE_NOMINMAX +#endif +#include +#ifdef SIMPLNX_TEST_RESTORE_NOMINMAX +#undef SIMPLNX_TEST_RESTORE_NOMINMAX +#undef NOMINMAX +#endif +#endif + +namespace nx::core::UnitTest +{ +/** + * @brief Isolates one test process without taking ownership of borrowed fixtures. + * Routing remains installed after finish and destruction. Call before framework or Application construction. + * The ownership checks protect cooperative test processes, not hostile concurrent filesystem changes. + */ +class ProcessTestProfile +{ +public: + ProcessTestProfile() = default; + ProcessTestProfile(const ProcessTestProfile&) = delete; + ProcessTestProfile(ProcessTestProfile&&) = delete; + ProcessTestProfile& operator=(const ProcessTestProfile&) = delete; + ProcessTestProfile& operator=(ProcessTestProfile&&) = delete; + + ~ProcessTestProfile() noexcept + { + if(!m_Finished && !m_Root.empty()) + { + try + { + std::cerr << "Unfinished test profile retained: " << m_Root << '\n'; + } catch(...) + { + } + } + } + + /** + * @brief Installs a fresh profile or validates an explicit marker and token request. + * @param error Receives a startup error. No ambient fallback is permitted. + * @return True when routing and optional external reporting succeed. + */ + bool install(std::string& error) + { + try + { + if(m_Attempted) + { + throw std::runtime_error("Test profile installation was already attempted"); + } + m_Attempted = true; + const char* requested = std::getenv("SIMPLNX_TEST_PROFILE_DIR"); + const char* token = std::getenv("SIMPLNX_TEST_PROFILE_TOKEN"); + if(requested != nullptr || token != nullptr) + { + if(requested == nullptr || token == nullptr || *requested == '\0' || *token == '\0') + { + throw std::runtime_error("Test profile directory and nonempty token must be supplied together"); + } + std::filesystem::path requestedPath(requested); + while(requestedPath != requestedPath.root_path() && (requestedPath.filename().empty() || requestedPath.filename() == ".")) + { + requestedPath = requestedPath.parent_path(); + } + if(!requestedPath.is_absolute() || requestedPath == requestedPath.root_path()) + { + throw std::runtime_error("Borrowed test profile must be an absolute non-root directory"); + } + // Ancestor aliases can name a valid fixture. The requested directory itself must remain a plain entry. + requirePlainEntry(requestedPath); + m_Root = std::filesystem::canonical(requestedPath); + if(m_Root == m_Root.root_path()) + { + throw std::runtime_error("Borrowed test profile resolves to a filesystem root"); + } + requirePlainPath(m_Root); + m_Token = token; + validateMarker(); + } + else + { + m_Parent = std::filesystem::canonical(std::filesystem::temp_directory_path()); + requirePlainPath(m_Parent); + std::random_device random; + for(int attempt = 0; attempt < 32; ++attempt) + { + m_Token = std::to_string(random()) + "-" + std::to_string(random()) + "-" + std::to_string(random()); + const auto candidate = m_Parent / ("simplnx-test-profile-" + m_Token); + if(std::filesystem::create_directory(candidate)) + { + m_Root = candidate; + m_OwnsRoot = true; + break; + } + } + if(m_Root.empty()) + { + throw std::runtime_error("Cannot reserve an exclusive test profile directory"); + } + writeFile(m_Root / ".simplnx-test-profile", m_Token); + std::filesystem::create_directories(corePreferencesFile().parent_path()); + writeFile(corePreferencesFile(), "{}"); + } + + // Core Preferences reads the narrow CRT environment on Windows. + const auto rootString = m_Root.string(); + if(std::filesystem::path(rootString) != m_Root) + { + throw std::runtime_error("Test profile path cannot round-trip through the core environment"); + } + setEnvironment(homeVariable(), rootString.c_str()); + const char* installed = std::getenv(homeVariable()); + if(installed == nullptr || std::filesystem::path(installed) != m_Root) + { + throw std::runtime_error("Core environment did not retain the test profile path"); + } +#ifdef _WIN32 + std::wstring nativeHome(32768, L'\0'); + const DWORD size = GetEnvironmentVariableW(L"USERPROFILE", nativeHome.data(), static_cast(nativeHome.size())); + if(size == 0 || size >= nativeHome.size()) + { + throw std::runtime_error("Inherited Windows environment has no usable test profile"); + } + nativeHome.resize(size); + if(std::filesystem::path(nativeHome) != m_Root) + { + throw std::runtime_error("CRT and inherited Windows profile paths differ"); + } +#endif + setEnvironment("SIMPLNX_TEST_PROFILE_DIR", nullptr); + setEnvironment("SIMPLNX_TEST_PROFILE_TOKEN", nullptr); + setEnvironment("SIMPLNX_TEST_ACTIVE_PROFILE", rootString.c_str()); + if(const char* report = std::getenv("SIMPLNX_TEST_PROFILE_REPORT")) + { + const std::filesystem::path reportPath(report); + setEnvironment("SIMPLNX_TEST_PROFILE_REPORT", nullptr); + if(!reportPath.is_absolute()) + { + throw std::runtime_error("Test profile report must use an absolute external path"); + } + const auto reportParent = std::filesystem::canonical(reportPath.parent_path()); + requirePlainPath(reportParent); + const auto resolvedReport = reportParent / reportPath.filename(); + for(auto parent = reportParent; !parent.empty(); parent = parent.parent_path()) + { + if(parent == m_Root) + { + throw std::runtime_error("Test profile report must be outside the profile root"); + } + if(parent == parent.root_path()) + { + break; + } + } + if(std::filesystem::exists(resolvedReport) || std::filesystem::is_symlink(std::filesystem::symlink_status(resolvedReport))) + { + throw std::runtime_error("Test profile report already exists"); + } + writeFile(resolvedReport, rootString); + } + m_Installed = true; + error.clear(); + return true; + } catch(const std::exception& exception) + { + error = exception.what(); + return false; + } + } + + const std::filesystem::path& root() const noexcept + { + return m_Root; + } + bool ownsRoot() const noexcept + { + return m_OwnsRoot; + } + + std::filesystem::path corePreferencesFile() const + { +#ifdef _WIN32 + return m_Root / "AppData/Local/DREAM3DNX/preferences.json"; +#elif defined(__APPLE__) + return m_Root / "Library/Preferences/DREAM3DNX/preferences.json"; +#else + return m_Root / ".config/DREAM3DNX/preferences.json"; +#endif + } + + /** + * @brief Removes only a successfully completed, exclusively owned profile. + * @param successful True after the framework and all known owners finish successfully. + * @param error Receives an error and the retained or residual path. + * @return The cached completion result. Borrowed roots are never removed or repaired. + */ + bool finish(bool successful, std::string& error) noexcept + { + try + { + if(!m_Finished) + { + m_Finished = true; + if(!successful || !m_Installed) + { + throw std::runtime_error("Test did not complete successfully"); + } + if(m_OwnsRoot) + { + requirePlainPath(m_Root); + if(m_Root.parent_path() != m_Parent || m_Root.filename() != "simplnx-test-profile-" + m_Token || std::filesystem::canonical(m_Root) != m_Root) + { + throw std::runtime_error("Owned test profile path changed"); + } + validateMarker(); + // Nested profile markers and links can belong to independent or active owners. + for(const auto& entry : std::filesystem::recursive_directory_iterator(m_Root)) + { + requirePlainEntry(entry.path()); + if(entry.path().filename() == ".simplnx-test-profile" && entry.path().parent_path() != m_Root) + { + throw std::runtime_error("Independent test profile remains below the owned root"); + } + } + std::error_code removeError; + std::filesystem::remove_all(m_Root, removeError); + if(removeError) + { + throw std::runtime_error("Cannot remove owned test profile: " + removeError.message()); + } + } + m_FinishSucceeded = true; + } + error = m_FinishError; + return m_FinishSucceeded; + } catch(const std::exception& exception) + { + try + { + m_FinishError = std::string(exception.what()) + "; test profile retained or partially removed: " + m_Root.string(); + error = m_FinishError; + } catch(...) + { + } + return false; + } catch(...) + { + return false; + } + } + + /** + * @brief Finishes the profile and combines its outcome with a framework exit code. + * @param result Framework result, including successful listing counts above 255. + * @param successful True when the framework and known owners completed successfully. + * @param error Receives the completion error. + * @return Original result on success; a failure in 1 through 255 otherwise. + */ + int finishExitCode(int result, bool successful, std::string& error) noexcept + { + if(finish(successful, error)) + { + return result; + } + // POSIX keeps only eight exit-status bits. Large listing counts must not hide cleanup failures. + return result > 0 && result <= 255 ? result : 1; + } + + static const char* homeVariable() noexcept + { +#ifdef _WIN32 + return "USERPROFILE"; +#else + return "HOME"; +#endif + } + +private: + static void setEnvironment(const char* name, const char* value) + { +#ifdef _WIN32 + const int result = _putenv_s(name, value == nullptr ? "" : value); +#else + const int result = value == nullptr ? unsetenv(name) : setenv(name, value, 1); +#endif + if(result != 0) + { + throw std::runtime_error(std::string("Cannot set test environment variable: ") + name); + } + } + + static void writeFile(const std::filesystem::path& path, const std::string& text) + { + std::ofstream stream(path, std::ios::binary); + stream << text; + stream.close(); + if(!stream) + { + throw std::runtime_error("Cannot write test profile file: " + path.string()); + } + } + + static void requirePlainEntry(const std::filesystem::path& path) + { + const auto status = std::filesystem::symlink_status(path); + if(!std::filesystem::is_directory(status) && !std::filesystem::is_regular_file(status)) + { + throw std::runtime_error("Test profile contains a missing path, link or special file: " + path.string()); + } +#ifdef _WIN32 + const DWORD attributes = GetFileAttributesW(path.c_str()); + if(attributes == INVALID_FILE_ATTRIBUTES || (attributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) + { + throw std::runtime_error("Test profile contains a reparse point or unreadable path: " + path.string()); + } +#endif + } + + static void requirePlainPath(const std::filesystem::path& path) + { + for(auto parent = path; !parent.empty(); parent = parent.parent_path()) + { + requirePlainEntry(parent); + if(parent == parent.root_path()) + { + break; + } + } + } + + void validateMarker() const + { + if(!std::filesystem::is_directory(m_Root)) + { + throw std::runtime_error("Test profile is not a directory"); + } + const auto marker = m_Root / ".simplnx-test-profile"; + requirePlainEntry(marker); + if(!std::filesystem::is_regular_file(marker) || std::filesystem::file_size(marker) != m_Token.size()) + { + throw std::runtime_error("Test profile marker is not a matching regular file"); + } + std::ifstream stream(marker, std::ios::binary); + const std::string actual((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + if(!stream || actual != m_Token) + { + throw std::runtime_error("Test profile marker token does not match"); + } + } + + std::filesystem::path m_Root; + std::filesystem::path m_Parent; + std::string m_Token; + std::string m_FinishError; + bool m_Attempted = false; + bool m_Installed = false; + bool m_OwnsRoot = false; + bool m_Finished = false; + bool m_FinishSucceeded = false; +}; +} // namespace nx::core::UnitTest diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/ScopedDefaultFormatResolver.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/ScopedDefaultFormatResolver.hpp new file mode 100644 index 0000000000..acaf96e6ba --- /dev/null +++ b/test/UnitTestCommon/include/simplnx/UnitTest/ScopedDefaultFormatResolver.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "simplnx/Common/ScopeGuard.hpp" +#include "simplnx/DataStructure/DataStructure.hpp" + +#include +#include + +namespace nx::core::UnitTest +{ +/** + * @brief Temporarily replaces the process-wide storage-format resolver. + * @param resolver Replacement policy. A null pointer leaves the current policy active. + * @return Guard that restores the previous shared owner on normal exit or exception unwinding. + * @pre No concurrent default-resolver access or mutation occurs during the guarded scope. + */ +[[nodiscard]] inline auto SetDefaultFormatResolverForScope(std::shared_ptr resolver) +{ + auto previousResolver = DataStructure::getDefaultFormatResolver(); + auto restoreResolver = MakeScopeGuard([savedResolver = std::move(previousResolver)]() noexcept { DataStructure::setDefaultFormatResolver(savedResolver); }); + DataStructure::setDefaultFormatResolver(std::move(resolver)); + return restoreResolver; +} +} // namespace nx::core::UnitTest diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/TestApplicationCleanup.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/TestApplicationCleanup.hpp new file mode 100644 index 0000000000..8edf66a0f0 --- /dev/null +++ b/test/UnitTestCommon/include/simplnx/UnitTest/TestApplicationCleanup.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include "simplnx/Core/Application.hpp" + +#include +#include +#include + +namespace nx::core::UnitTest +{ +/** + * @brief Checks the existing Application save, deletes the singleton, and checks for retained owners. + * @return True if the checked save and deletion complete without errors and no observed owner remains. + * @pre Known framework, worker, and UI owners have released Application under the required runtime scopes. + * @note Does not construct Application or control Qt, Python, backend, or profile lifetimes. + * @note Caught save errors do not skip deletion. A terminating noexcept destructor cannot be recovered here. + */ +inline bool FinishTestApplication() noexcept +{ + bool successful = true; + std::weak_ptr application; + try + { + if(auto app = Application::Instance()) + { + application = app; + const auto saveResult = app->savePreferences(); + if(saveResult.invalid()) + { + successful = false; + for(const auto& saveError : saveResult.errors()) + { + std::fprintf(stderr, "Test Application save failed: %s\n", saveError.message.c_str()); + } + } + } + } catch(const std::exception& exception) + { + std::fprintf(stderr, "Test Application save failed: %s\n", exception.what()); + successful = false; + } catch(...) + { + std::fputs("Test Application save failed with an unknown exception\n", stderr); + successful = false; + } + + // The temporary strong owner leaves scope before deletion and the weak-owner check. + // DeleteInstance and the destructor report additional save failures through logging only. + try + { + Application::DeleteInstance(); + } catch(const std::exception& exception) + { + std::fprintf(stderr, "Test Application deletion failed: %s\n", exception.what()); + successful = false; + } catch(...) + { + std::fputs("Test Application deletion failed with an unknown exception\n", stderr); + successful = false; + } + if(!application.expired()) + { + std::fputs("Application owner remains after test framework teardown\n", stderr); + successful = false; + } + return successful; +} +} // namespace nx::core::UnitTest diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.cpp b/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.cpp index a9c7f504f3..15a3684d09 100644 --- a/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.cpp +++ b/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace nx::core::UnitTest { @@ -288,12 +289,25 @@ std::error_code TestFileSentinel::decompress() } PreferencesSentinel::PreferencesSentinel(nx::core::DataStorageMode mode, int64 largeDataSize) +: PreferencesSentinel(mode, nlohmann::json(largeDataSize)) +{ +} + +PreferencesSentinel::PreferencesSentinel(nx::core::DataStorageMode mode, const nlohmann::json& largeDataSize) { auto* prefs = nx::core::Application::Instance()->getPreferences(); + const auto validation = nx::core::Preferences::ValidateOocSizeValue(nx::core::Preferences::k_LargeDataSize_Key, largeDataSize); + if(validation.invalid()) + { + throw std::invalid_argument(validation.errors().front().message); + } + // Save both values before this sentinel changes the process-wide preferences. - m_OriginalMode = prefs->dataStorageMode(); - m_OriginalSize = prefs->valueAs(nx::core::Preferences::k_LargeDataSize_Key); + m_HadMode = prefs->contains(nx::core::Preferences::k_DataStorageMode_Key); + m_OriginalMode = prefs->value(nx::core::Preferences::k_DataStorageMode_Key); + m_HadSize = prefs->contains(nx::core::Preferences::k_LargeDataSize_Key); + m_OriginalSize = prefs->value(nx::core::Preferences::k_LargeDataSize_Key); // Apply the test values only after the complete prior state is available. prefs->setDataStorageMode(mode); @@ -307,8 +321,22 @@ PreferencesSentinel::~PreferencesSentinel() // Restore only the in-memory values. Unit tests must not write the developer's preferences file. // A concurrent save or a terminated test could persist a temporary storage mode. // Later test processes would then inherit that incorrect mode. - prefs->setDataStorageMode(m_OriginalMode); - prefs->setValue(nx::core::Preferences::k_LargeDataSize_Key, m_OriginalSize); + if(m_HadSize) + { + prefs->setValue(nx::core::Preferences::k_LargeDataSize_Key, m_OriginalSize); + } + else + { + prefs->removeValue(nx::core::Preferences::k_LargeDataSize_Key); + } + if(m_HadMode) + { + prefs->setValue(nx::core::Preferences::k_DataStorageMode_Key, m_OriginalMode); + } + else + { + prefs->removeValue(nx::core::Preferences::k_DataStorageMode_Key); + } } } // namespace nx::core::UnitTest diff --git a/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp b/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp index 2d8780ade8..b10bb0dd8b 100644 --- a/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp +++ b/test/UnitTestCommon/include/simplnx/UnitTest/UnitTestCommon.hpp @@ -35,14 +35,20 @@ #include "simplnx/Utilities/Parsing/DREAM3D/Dream3dIO.hpp" #include "simplnx/Utilities/Parsing/HDF5/IO/FileIO.hpp" +#include + #include #include #include #include +#include #include +#include +#include #include +#include #include namespace fs = std::filesystem; @@ -422,6 +428,14 @@ class PreferencesSentinel */ PreferencesSentinel(nx::core::DataStorageMode mode, int64 largeDataSize); + /** + * @brief Applies a temporary threshold without narrowing its JSON integer type. + * @param mode Storage mode to use during the sentinel lifetime. + * @param largeDataSize Exact JSON integer + * threshold in bytes. + */ + PreferencesSentinel(nx::core::DataStorageMode mode, const nlohmann::json& largeDataSize); + /** * @brief Restores the saved storage preferences without writing a preferences file. */ @@ -433,8 +447,10 @@ class PreferencesSentinel PreferencesSentinel& operator=(PreferencesSentinel&&) = delete; private: - nx::core::DataStorageMode m_OriginalMode; - int64 m_OriginalSize; + nlohmann::json m_OriginalMode; + nlohmann::json m_OriginalSize; + bool m_HadMode = false; + bool m_HadSize = false; }; /** @@ -818,32 +834,63 @@ void CompareArrays(const DataStructure& dataStructure, const DataPath& exemplary } /** - * @brief Compares two floating-point DataArray objects and reports their largest difference. - * @tparam T Specifies the floating-point element type. - * @param dataStructure Contains both arrays. - * @param exemplaryDataPath Exemplar array path. - * @param computedPath Computed array path. - * @param epsilon Exclusive absolute-difference limit. - * @param checkNans True to compare NaN positions; false to omit values where either array has NaN. + * @namespace FloatArrayComparisonDetail + * @brief Contains internal test comparison support. + */ +namespace FloatArrayComparisonDetail +{ +/** + * @struct FloatArrayComparison + * @brief Separates numerical differences from errors that prevent a complete comparison. + * @tparam T Floating-point element type used for difference calculations. */ template -void CompareFloatArraysWithNans(const DataStructure& dataStructure, const DataPath& exemplaryDataPath, const DataPath& computedPath, const T& epsilon = EPSILON, bool checkNans = true) +struct FloatArrayComparison { static_assert(std::is_floating_point_v); - REQUIRE_NOTHROW(dataStructure.getDataRefAs>(exemplaryDataPath)); - REQUIRE_NOTHROW(dataStructure.getDataRefAs>(computedPath)); - - const auto& exemplaryDataArray = dataStructure.getDataRefAs>(exemplaryDataPath); - const auto& generatedDataArray = dataStructure.getDataRefAs>(computedPath); - REQUIRE(generatedDataArray.getNumberOfTuples() == exemplaryDataArray.getNumberOfTuples()); + bool matches = false; + T maximumDifference = 0; + usize maximumDifferenceIndex = 0; + std::optional firstNanMismatchIndex; + T exemplaryNanMismatchValue = {}; + T generatedNanMismatchValue = {}; +}; - INFO(fmt::format("Input Data Array:'{}' Output DataArray: '{}' bad comparison", exemplaryDataPath.toString(), computedPath.toString())); +/** + * @brief Compares all values without making Catch assertions. + * @tparam T Floating-point element type used for difference calculations. + * @param exemplaryArray Exemplar values. + * @param generatedArray Computed values. + * @param epsilon Exclusive absolute-difference limit. + * @param checkNans True to compare NaN positions; false to omit pairs that contain NaN. + * @return Comparison diagnostics, or an error for unequal sizes or a failed read. + */ +template +Result> EvaluateFloatArraysWithNans(const DataArray& exemplaryArray, const DataArray& generatedArray, const T& epsilon, bool checkNans) +{ + static_assert(std::is_floating_point_v); + if(exemplaryArray.getNumberOfTuples() != generatedArray.getNumberOfTuples()) + { + return MakeErrorResult>( + -90051, fmt::format("Floating-array comparison requires equal tuple counts: exemplar {}, computed {}.", exemplaryArray.getNumberOfTuples(), generatedArray.getNumberOfTuples())); + } + if(exemplaryArray.getNumberOfComponents() != generatedArray.getNumberOfComponents()) + { + return MakeErrorResult>( + -90052, fmt::format("Floating-array comparison requires equal component counts: exemplar {}, computed {}.", exemplaryArray.getNumberOfComponents(), generatedArray.getNumberOfComponents())); + } - const auto& oldStore = exemplaryDataArray.getDataStoreRef(); - const auto& newStore = generatedDataArray.getDataStoreRef(); + const auto& oldStore = exemplaryArray.getDataStoreRef(); + const auto& newStore = generatedArray.getDataStoreRef(); const usize totalSize = oldStore.getSize(); + if(totalSize != newStore.getSize()) + { + return MakeErrorResult>(-90053, fmt::format("Floating-array comparison requires equal flat sizes: exemplar {}, computed {}.", totalSize, newStore.getSize())); + } + Result> result; + auto& comparison = result.value(); constexpr usize k_ChunkSize = 40000; auto oldBuf = std::make_unique(k_ChunkSize); auto newBuf = std::make_unique(k_ChunkSize); @@ -851,43 +898,85 @@ void CompareFloatArraysWithNans(const DataStructure& dataStructure, const DataPa // The full scan reports the largest difference for the V&V diagnostic. // Bounded bulk reads keep memory independent of the total element count. // The difference uses T to preserve the input array precision. - T maxDiff = 0; - usize maxDiffIndex = 0; - for(usize offset = 0; offset < totalSize; offset += k_ChunkSize) { const usize count = std::min(k_ChunkSize, totalSize - offset); auto oldStoreReadResult = oldStore.copyIntoBuffer(offset, nonstd::span(oldBuf.get(), count)); - SIMPLNX_RESULT_REQUIRE_VALID(oldStoreReadResult); + result.warnings().insert(result.warnings().end(), oldStoreReadResult.warnings().begin(), oldStoreReadResult.warnings().end()); + if(oldStoreReadResult.invalid()) + { + result.m_Expected = nonstd::make_unexpected(std::move(oldStoreReadResult.errors())); + return result; + } auto newStoreReadResult = newStore.copyIntoBuffer(offset, nonstd::span(newBuf.get(), count)); - SIMPLNX_RESULT_REQUIRE_VALID(newStoreReadResult); + result.warnings().insert(result.warnings().end(), newStoreReadResult.warnings().begin(), newStoreReadResult.warnings().end()); + if(newStoreReadResult.invalid()) + { + result.m_Expected = nonstd::make_unexpected(std::move(newStoreReadResult.errors())); + return result; + } - for(usize i = 0; i < count; i++) + for(usize valueIndex = 0; valueIndex < count; ++valueIndex) { - const T oldVal = oldBuf[i]; - const T newVal = newBuf[i]; - if(!checkNans && (std::isnan(newVal) || std::isnan(oldVal))) + const T oldVal = oldBuf[valueIndex]; + const T newVal = newBuf[valueIndex]; + const bool oldIsNan = std::isnan(oldVal); + const bool newIsNan = std::isnan(newVal); + if(oldIsNan || newIsNan) { - continue; - } - if(std::isnan(oldVal) && std::isnan(newVal)) - { - // Two NaN values represent the same missing result for this test helper. + if(checkNans && oldIsNan != newIsNan && !comparison.firstNanMismatchIndex.has_value()) + { + comparison.firstNanMismatchIndex = offset + valueIndex; + comparison.exemplaryNanMismatchValue = oldVal; + comparison.generatedNanMismatchValue = newVal; + } + // Paired NaNs represent the same missing result. A one-sided NaN does not stop the full diagnostic scan. continue; } if(oldVal != newVal) { - const T diff = std::fabs(static_cast(oldVal - newVal)); - if(diff > maxDiff) + const T difference = std::fabs(static_cast(oldVal - newVal)); + if(difference > comparison.maximumDifference) { - maxDiff = diff; - maxDiffIndex = offset + i; + comparison.maximumDifference = difference; + comparison.maximumDifferenceIndex = offset + valueIndex; } } } } - INFO(fmt::format("Maximum difference of {} occurs at index {} (epsilon = {})", maxDiff, maxDiffIndex, epsilon)); - REQUIRE(maxDiff < epsilon); + comparison.matches = !comparison.firstNanMismatchIndex.has_value() && comparison.maximumDifference < epsilon; + return result; +} +} // namespace FloatArrayComparisonDetail + +/** + * @brief Compares two floating-point DataArray objects and reports their largest difference. + * @tparam T Specifies the floating-point element type. + * @param dataStructure Contains both arrays. + * @param exemplaryDataPath Exemplar array path. + * @param computedPath Computed array path. + * @param epsilon Exclusive absolute-difference limit. + * @param checkNans True to compare NaN positions; false to omit values where either array has NaN. + */ +template +void CompareFloatArraysWithNans(const DataStructure& dataStructure, const DataPath& exemplaryDataPath, const DataPath& computedPath, const T& epsilon = EPSILON, bool checkNans = true) +{ + static_assert(std::is_floating_point_v); + INFO(fmt::format("Input Data Array: '{}' Output DataArray: '{}'", exemplaryDataPath.toString(), computedPath.toString())); + REQUIRE_NOTHROW(dataStructure.getDataRefAs>(exemplaryDataPath)); + REQUIRE_NOTHROW(dataStructure.getDataRefAs>(computedPath)); + const auto& exemplaryDataArray = dataStructure.getDataRefAs>(exemplaryDataPath); + const auto& generatedDataArray = dataStructure.getDataRefAs>(computedPath); + auto evaluation = FloatArrayComparisonDetail::EvaluateFloatArraysWithNans(exemplaryDataArray, generatedDataArray, epsilon, checkNans); + SIMPLNX_RESULT_REQUIRE_VALID(evaluation); + const auto& comparison = evaluation.value(); + INFO(fmt::format("Maximum difference of {} occurs at index {} (epsilon = {})", comparison.maximumDifference, comparison.maximumDifferenceIndex, epsilon)); + if(comparison.firstNanMismatchIndex.has_value()) + { + UNSCOPED_INFO( + fmt::format("NaN positions differ at index {}: exemplar {}, computed {}.", *comparison.firstNanMismatchIndex, comparison.exemplaryNanMismatchValue, comparison.generatedNanMismatchValue)); + } + REQUIRE(comparison.matches); } /** diff --git a/test/VerifyTestProfileIsolation.cmake b/test/VerifyTestProfileIsolation.cmake new file mode 100644 index 0000000000..a9fcf0cbc5 --- /dev/null +++ b/test/VerifyTestProfileIsolation.cmake @@ -0,0 +1,79 @@ +if(NOT DEFINED PROFILE_TEST_EXECUTABLE OR NOT EXISTS "${PROFILE_TEST_EXECUTABLE}") + message(FATAL_ERROR "PROFILE_TEST_EXECUTABLE must name an existing Catch executable") +endif() +if(NOT DEFINED PROFILE_TEST_ROOT OR NOT IS_ABSOLUTE "${PROFILE_TEST_ROOT}") + message(FATAL_ERROR "PROFILE_TEST_ROOT must name an absolute test artifact directory") +endif() + +# Each invocation owns a new fixture. Failed runs retain it for diagnosis. +string(RANDOM LENGTH 24 ALPHABET 0123456789abcdef nonce) +set(case_root "${PROFILE_TEST_ROOT}/sentinel-${nonce}") +if(EXISTS "${case_root}") + message(FATAL_ERROR "Refusing to reuse an existing sentinel fixture: ${case_root}") +endif() +set(fake_home "${case_root}/fake-ambient") +if(WIN32) + set(preferences_dir "${fake_home}/AppData/Local/DREAM3DNX") + set(home_variable "USERPROFILE=${fake_home}") +elseif(APPLE) + set(preferences_dir "${fake_home}/Library/Preferences/DREAM3DNX") + set(home_variable "HOME=${fake_home}") +else() + set(preferences_dir "${fake_home}/.config/DREAM3DNX") + set(home_variable "HOME=${fake_home}") +endif() +file(MAKE_DIRECTORY "${preferences_dir}") +set(preferences_file "${preferences_dir}/preferences.json") +# The legacy byte threshold proves that the test does not save, and so convert, the ambient file. +set(expected_json "{\"large_data_size\":123456789,\"plugins\":{},\"profile_isolation_sentinel\":\"${nonce}\"}") +file(WRITE "${preferences_file}" "${expected_json}") +file(WRITE "${case_root}/expected-preferences.json" "${expected_json}") +file(WRITE "${case_root}/.profile-driver" "${nonce}") + +# Redirect only the child. Never read or write the developer's ambient profile. +# Inherited test requests must not select an unrelated caller-owned fixture. +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + --unset=SIMPLNX_TEST_PROFILE_DIR + --unset=SIMPLNX_TEST_PROFILE_TOKEN + --unset=SIMPLNX_TEST_ACTIVE_PROFILE + --unset=SIMPLNX_TEST_PROFILE_REPORT + "${home_variable}" + "${PROFILE_TEST_EXECUTABLE}" "Application::savePreferences" --reporter compact --warn NoTests + WORKING_DIRECTORY "${case_root}" + RESULT_VARIABLE result + OUTPUT_VARIABLE output + ERROR_VARIABLE error + TIMEOUT 120 +) +file(WRITE "${case_root}/stdout.txt" "${output}") +file(WRITE "${case_root}/stderr.txt" "${error}") +if(NOT result STREQUAL "0") + message(FATAL_ERROR "Application save case failed with '${result}'. Evidence: ${case_root}\n${output}\n${error}") +endif() + +if(NOT output MATCHES "Passed 1 test case") + message(FATAL_ERROR "Expected exactly one passing Application save case. Evidence: ${case_root}\n${output}") +endif() + +file(READ "${preferences_file}" actual_json) +if(NOT actual_json STREQUAL expected_json) + string(JSON written_value ERROR_VARIABLE witness_error GET "${actual_json}" test_key) + message(FATAL_ERROR + "Application::savePreferences changed the fake ambient preference bytes. " + "The existing save case writes test_key=42; observed '${written_value}' (JSON status '${witness_error}'). " + "Evidence: ${case_root}\nExpected: ${expected_json}\nActual: ${actual_json}") +endif() + +# The driver removes only its exact fixture after the child and byte comparison succeed. +file(REAL_PATH "${PROFILE_TEST_ROOT}" resolved_parent) +file(REAL_PATH "${case_root}" resolved_root) +get_filename_component(actual_parent "${resolved_root}" DIRECTORY) +file(READ "${case_root}/.profile-driver" actual_token) +if(NOT actual_parent STREQUAL resolved_parent OR NOT actual_token STREQUAL nonce OR IS_SYMLINK "${case_root}") + message(FATAL_ERROR "Sentinel fixture ownership changed: ${case_root}") +endif() +file(REMOVE_RECURSE "${resolved_root}") +if(EXISTS "${resolved_root}") + message(FATAL_ERROR "Cannot remove completed sentinel fixture: ${resolved_root}") +endif() diff --git a/test/VerifyTestProfileLifecycle.cmake b/test/VerifyTestProfileLifecycle.cmake new file mode 100644 index 0000000000..4155b6af5f --- /dev/null +++ b/test/VerifyTestProfileLifecycle.cmake @@ -0,0 +1,326 @@ +set(required_executables PROFILE_TEST_EXECUTABLE) +if(NOT PROFILE_RUNNER_ONLY) + list(APPEND required_executables PROFILE_PROBE_EXECUTABLE) +endif() +foreach(required IN LISTS required_executables) + if(NOT DEFINED ${required} OR NOT EXISTS "${${required}}") + message(FATAL_ERROR "${required} must name an existing executable") + endif() +endforeach() +if(NOT DEFINED PROFILE_TEST_ROOT OR NOT IS_ABSOLUTE "${PROFILE_TEST_ROOT}") + message(FATAL_ERROR "PROFILE_TEST_ROOT must be absolute") +endif() +string(RANDOM LENGTH 24 ALPHABET 0123456789abcdef nonce) +set(case_root "${PROFILE_TEST_ROOT}/lifecycle-${nonce}") +if(EXISTS "${case_root}") + message(FATAL_ERROR "Refusing to reuse ${case_root}") +endif() +file(MAKE_DIRECTORY "${case_root}/temp" "${case_root}/fake-ambient") +file(REAL_PATH "${case_root}" case_root) +file(WRITE "${case_root}/.profile-driver" "${nonce}") +file(WRITE "${case_root}/shared-artifact.txt" "caller-owned artifact") +if(WIN32) + set(preferences_suffix "AppData/Local/DREAM3DNX/preferences.json") + set(home_variable "USERPROFILE") +elseif(APPLE) + set(preferences_suffix "Library/Preferences/DREAM3DNX/preferences.json") + set(home_variable "HOME") +else() + set(preferences_suffix ".config/DREAM3DNX/preferences.json") + set(home_variable "HOME") +endif() +set(fake_preferences "${case_root}/fake-ambient/${preferences_suffix}") +get_filename_component(fake_preferences_dir "${fake_preferences}" DIRECTORY) +file(MAKE_DIRECTORY "${fake_preferences_dir}") +set(sentinel "{\"plugins\":{},\"sentinel\":\"${nonce}\"}") +file(WRITE "${fake_preferences}" "${sentinel}") + +# The driver owns its temporary parent. The generic helper must leave these variables unchanged. +set(ENV{TEMP} "${case_root}/temp") +set(ENV{TMP} "${case_root}/temp") +set(ENV{TMPDIR} "${case_root}/temp") +set(ENV{${home_variable}} "${case_root}/fake-ambient") +set(borrowed_env) +set(expected_borrowed "") +set(report_parent "${case_root}") + +function(run_case name executable expected_status disposition) + set(report "${report_parent}/${name}.profile") + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + --unset=SIMPLNX_TEST_PROFILE_DIR --unset=SIMPLNX_TEST_PROFILE_TOKEN + "SIMPLNX_TEST_PROFILE_REPORT=${report}" ${borrowed_env} + "${executable}" ${ARGN} + WORKING_DIRECTORY "${case_root}" + RESULT_VARIABLE status OUTPUT_VARIABLE output ERROR_VARIABLE error TIMEOUT 20) + file(WRITE "${case_root}/${name}.stdout" "${output}") + file(WRITE "${case_root}/${name}.stderr" "${error}") + if(expected_status STREQUAL "list") + string(REGEX REPLACE "[^\n]" "" newlines "${output}") + string(LENGTH "${newlines}" listed_count) + if(listed_count LESS 256) + message(FATAL_ERROR "Full discovery must cover more than 255 cases, found ${listed_count}") + endif() + set(expected_status "${listed_count}") + if(NOT WIN32) + math(EXPR expected_status "${listed_count} % 256") + endif() + if(NOT error STREQUAL "") + message(FATAL_ERROR "Full discovery produced unexpected stderr: ${error}") + endif() + endif() + if(NOT status STREQUAL "${expected_status}") + message(FATAL_ERROR "${name}: status '${status}', expected '${expected_status}'. Evidence: ${case_root}\n${output}\n${error}") + endif() + file(READ "${fake_preferences}" actual_sentinel) + file(READ "${case_root}/shared-artifact.txt" actual_artifact) + if(NOT actual_sentinel STREQUAL sentinel OR NOT actual_artifact STREQUAL "caller-owned artifact") + message(FATAL_ERROR "${name} changed external artifacts. Evidence: ${case_root}") + endif() + if(disposition STREQUAL "rejected") + if(EXISTS "${report}") + message(FATAL_ERROR "${name} installed a rejected profile") + endif() + return() + endif() + if(NOT EXISTS "${report}") + message(FATAL_ERROR "${name} did not report its profile. Evidence: ${case_root}") + endif() + file(READ "${report}" active) + file(TO_CMAKE_PATH "${active}" active) + if(NOT IS_ABSOLUTE "${active}") + message(FATAL_ERROR "${name} reported a relative profile") + endif() + get_filename_component(parent "${active}" DIRECTORY) + get_filename_component(leaf "${active}" NAME) + if(expected_borrowed STREQUAL "") + if(NOT parent STREQUAL "${case_root}/temp" OR NOT leaf MATCHES "^simplnx-test-profile-[0-9]+-[0-9]+-[0-9]+$") + message(FATAL_ERROR "${name} reported a profile outside the driver temporary parent: ${active}") + endif() + elseif(NOT active STREQUAL expected_borrowed) + message(FATAL_ERROR "${name} did not use the declared borrowed fixture: ${active}") + endif() + if(disposition STREQUAL "removed") + if(EXISTS "${active}") + message(FATAL_ERROR "${name} left a successful automatic profile: ${active}") + endif() + elseif(disposition STREQUAL "late") + file(READ "${active}/late-witness.txt" witness) + if(NOT witness STREQUAL "isolated after removal" OR EXISTS "${active}/.simplnx-test-profile") + message(FATAL_ERROR "Late routing did not recreate only its isolated witness: ${active}") + endif() + elseif(disposition STREQUAL "replaced") + file(READ "${active}/unrelated.txt" replacement) + if(NOT replacement STREQUAL "replacement must survive" OR NOT EXISTS "${active}-original/.simplnx-test-profile") + message(FATAL_ERROR "Replaced root or original ownership evidence was removed") + endif() + elseif(NOT EXISTS "${active}/.simplnx-test-profile") + message(FATAL_ERROR "${name} removed retained ownership evidence: ${active}") + else() + file(READ "${active}/.simplnx-test-profile" retained_token) + if(disposition STREQUAL "tampered") + set(expected_token "changed-token") + elseif(NOT expected_borrowed STREQUAL "") + set(expected_token "${nonce}") + else() + string(REGEX REPLACE "^simplnx-test-profile-" "" expected_token "${leaf}") + endif() + if(NOT retained_token STREQUAL expected_token) + message(FATAL_ERROR "${name} changed the retained ownership token") + endif() + endif() + set(last_profile "${active}" PARENT_SCOPE) + set(last_output "${output}" PARENT_SCOPE) + set(last_error "${error}" PARENT_SCOPE) +endfunction() + +if(NOT PROFILE_RUNNER_ONLY) + run_case(automatic "${PROFILE_PROBE_EXECUTABLE}" 0 removed success) + set(first_profile "${last_profile}") + set(ENV{SIMPLNX_TEST_ACTIVE_PROFILE} "${first_profile}") + run_case(independent "${PROFILE_PROBE_EXECUTABLE}" 0 removed success) + if(last_profile STREQUAL first_profile) + message(FATAL_ERROR "Inherited active metadata reused the first child profile") + endif() +endif() +run_case(round-trip "${PROFILE_TEST_EXECUTABLE}" 0 removed "Test profile saves and reloads an isolated witness" --reporter compact --warn NoTests) +if(NOT last_output MATCHES "Passed 1 test case") + message(FATAL_ERROR "Round-trip probe did not execute its real Catch case") +endif() +run_case(discovery "${PROFILE_TEST_EXECUTABLE}" 1 removed "Test profile saves and reloads an isolated witness" --list-test-names-only) +string(STRIP "${last_output}" listed) +if(NOT listed STREQUAL "Test profile saves and reloads an isolated witness" OR NOT last_error STREQUAL "") + message(FATAL_ERROR "Profile isolation changed discovery output: ${last_output}\n${last_error}") +endif() +run_case(help "${PROFILE_TEST_EXECUTABLE}" 0 removed --help) +if(NOT last_output MATCHES "--list-test-names-only" OR NOT last_error STREQUAL "") + message(FATAL_ERROR "Profile runner did not preserve Catch help output") +endif() +run_case(invalid-cli "${PROFILE_TEST_EXECUTABLE}" 255 retained --simplnx-nonexistent-test-option) +string(FIND "${last_error}" "Error(s) in input:" parse_error) +if(parse_error LESS 0) + message(FATAL_ERROR "Invalid CLI did not reach Catch argument validation") +endif() +# Catch opens the output stream while configuring the listing request. A regular-file +# ancestor forces that operation to throw without a test-only exception hook. +run_case(listing-error "${PROFILE_TEST_EXECUTABLE}" 255 retained --list-test-names-only --out "${case_root}/shared-artifact.txt/listing.txt") +if(NOT last_error MATCHES "Unable to open file:" OR NOT last_output STREQUAL "") + message(FATAL_ERROR "Listing failure did not reach Catch output-stream setup: ${last_output}\n${last_error}") +endif() +if(NOT PROFILE_RUNNER_ONLY) + run_case(full-discovery "${PROFILE_TEST_EXECUTABLE}" list removed --list-test-names-only) + # Reuse actual discovered cases instead of adding synthetic product registrations. + # This alphabet excludes Catch filter metacharacters, surrounding spaces, and CMake list separators. + file(STRINGS "${case_root}/full-discovery.stdout" boundary_candidates REGEX "^[A-Za-z0-9][A-Za-z0-9 _:/().+-]*[A-Za-z0-9_:/().+-]$") + list(LENGTH boundary_candidates boundary_candidate_count) + if(boundary_candidate_count LESS 256) + message(FATAL_ERROR "Listing boundary checks need 256 filter-safe existing names, found ${boundary_candidate_count}") + endif() + foreach(boundary_count IN ITEMS 255 256) + list(SUBLIST boundary_candidates 0 ${boundary_count} boundary_names) + list(JOIN boundary_names "\n" boundary_output) + string(APPEND boundary_output "\n") + set(boundary_input "${case_root}/listing-${boundary_count}.txt") + file(WRITE "${boundary_input}" "${boundary_output}") + set(boundary_status "${boundary_count}") + if(NOT WIN32) + math(EXPR boundary_status "${boundary_count} % 256") + endif() + run_case("listing-${boundary_count}" "${PROFILE_TEST_EXECUTABLE}" "${boundary_status}" removed --input-file "${boundary_input}" --list-test-names-only) + if(NOT last_output STREQUAL boundary_output OR NOT last_error STREQUAL "") + message(FATAL_ERROR "Listing ${boundary_count} did not preserve the exact selected names: ${last_output}\n${last_error}") + endif() + endforeach() + run_case(catch-failure "${PROFILE_TEST_EXECUTABLE}" 1 retained "Test profile retains a failed run" --reporter compact --warn NoTests) + if(NOT last_output MATCHES "Deliberate profile retention probe") + message(FATAL_ERROR "Catch failure did not reach the intended assertion") + endif() + run_case(retained-owner "${PROFILE_TEST_EXECUTABLE}" 1 retained "Test profile detects a retained Application owner" --reporter compact --warn NoTests) + if(NOT last_error MATCHES "Application owner remains") + message(FATAL_ERROR "Retained owner did not reach the lifetime check") + endif() + run_case(save-failure "${PROFILE_TEST_EXECUTABLE}" 1 retained "Test profile detects an Application save failure" --reporter compact --warn NoTests) + if(NOT last_error MATCHES "Test Application save failed") + message(FATAL_ERROR "Save failure did not reach the checked Application save") + endif() + run_case(failure "${PROFILE_PROBE_EXECUTABLE}" 8 retained failure) + run_case(unfinished "${PROFILE_PROBE_EXECUTABLE}" 9 retained unfinished) + run_case(tamper "${PROFILE_PROBE_EXECUTABLE}" 8 tampered tamper) + run_case(status-256-failure "${PROFILE_PROBE_EXECUTABLE}" 1 tampered status-256-failure) + if(NOT last_error MATCHES "Test profile marker") + message(FATAL_ERROR "Exit-status probe did not reach the controlled ownership failure") + endif() + set(success_256_status 0) + if(WIN32) + set(success_256_status 256) + endif() + run_case(status-256-success "${PROFILE_PROBE_EXECUTABLE}" "${success_256_status}" removed status-256-success) + run_case(nested-owner "${PROFILE_PROBE_EXECUTABLE}" 8 retained nested-owner) + run_case(root-replaced "${PROFILE_PROBE_EXECUTABLE}" 8 replaced root-replaced) + run_case(late "${PROFILE_PROBE_EXECUTABLE}" 0 late late) + + set(expected_borrowed "${case_root}/borrowed") + get_filename_component(borrowed_preferences_dir "${expected_borrowed}/${preferences_suffix}" DIRECTORY) + file(MAKE_DIRECTORY "${borrowed_preferences_dir}") + file(WRITE "${expected_borrowed}/.simplnx-test-profile" "${nonce}") + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${expected_borrowed}" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(borrowed-absent "${PROFILE_PROBE_EXECUTABLE}" 0 retained success) + if(EXISTS "${expected_borrowed}/${preferences_suffix}") + message(FATAL_ERROR "Bootstrap created an absent borrowed preferences file") + endif() + foreach(fixture IN ITEMS "{\"fixture\":31}" "malformed-json") + file(WRITE "${expected_borrowed}/${preferences_suffix}" "${fixture}") + string(MD5 fixture_id "${fixture}") + run_case("borrowed-${fixture_id}" "${PROFILE_PROBE_EXECUTABLE}" 0 retained success) + file(READ "${expected_borrowed}/${preferences_suffix}" actual) + if(NOT actual STREQUAL fixture) + message(FATAL_ERROR "Bootstrap changed a borrowed fixture") + endif() + endforeach() + run_case(borrowed-failure "${PROFILE_PROBE_EXECUTABLE}" 8 retained failure) + file(READ "${expected_borrowed}/${preferences_suffix}" after_failure) + if(NOT after_failure STREQUAL "malformed-json") + message(FATAL_ERROR "Failed borrowed completion changed its malformed fixture") + endif() + file(WRITE "${expected_borrowed}/${preferences_suffix}" "{\"plugins\":{},\"fixture\":31}") + run_case(borrowed-round-trip "${PROFILE_TEST_EXECUTABLE}" 0 retained "Test profile saves and reloads an isolated witness" --reporter compact --warn NoTests) + file(READ "${expected_borrowed}/${preferences_suffix}" saved) + string(JSON witness GET "${saved}" profile_round_trip) + string(JSON fixture GET "${saved}" fixture) + if(NOT witness STREQUAL "731" OR NOT fixture STREQUAL "31") + message(FATAL_ERROR "Borrowed Application save did not persist its witness and original fixture") + endif() + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${expected_borrowed}") + run_case(missing-token "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + set(borrowed_env "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(missing-directory "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=relative-profile" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(relative "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + get_filename_component(filesystem_root "${case_root}" ABSOLUTE) + cmake_path(GET filesystem_root ROOT_PATH filesystem_root) + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${filesystem_root}" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(root-path "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${expected_borrowed}" "SIMPLNX_TEST_PROFILE_TOKEN=wrong-token") + run_case(mismatched-token "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${case_root}/fake-ambient" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(missing-marker "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + file(MAKE_DIRECTORY "${case_root}/directory-marker/.simplnx-test-profile") + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${case_root}/directory-marker" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(directory-marker "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + file(CREATE_LINK "${expected_borrowed}" "${case_root}/linked-profile" SYMBOLIC RESULT link_result) + if(link_result STREQUAL "0") + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${case_root}/linked-profile" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(linked-profile "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${case_root}/linked-profile/." "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(linked-profile-trailing-dot "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + # Remove the link itself before recursive driver cleanup. + file(REMOVE "${case_root}/linked-profile") + else() + message(STATUS "Profile link rejection probe unavailable on this host: ${link_result}") + endif() + + file(MAKE_DIRECTORY "${case_root}/alias-target/borrowed") + file(CREATE_LINK "${case_root}/alias-target" "${case_root}/ancestor-alias" SYMBOLIC RESULT alias_result) + if(alias_result STREQUAL "0") + set(expected_borrowed "${case_root}/alias-target/borrowed") + file(WRITE "${expected_borrowed}/.simplnx-test-profile" "${nonce}") + get_filename_component(alias_preferences_dir "${expected_borrowed}/${preferences_suffix}" DIRECTORY) + file(MAKE_DIRECTORY "${alias_preferences_dir}") + file(WRITE "${expected_borrowed}/${preferences_suffix}" "{\"plugins\":{},\"alias_fixture\":91}") + set(report_parent "${case_root}/ancestor-alias") + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${case_root}/ancestor-alias/borrowed" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(alias-ancestor "${PROFILE_PROBE_EXECUTABLE}" 0 retained success) + run_case(alias-round-trip "${PROFILE_TEST_EXECUTABLE}" 0 retained "Test profile saves and reloads an isolated witness" --reporter compact --warn NoTests) + file(READ "${expected_borrowed}/${preferences_suffix}" saved_alias) + string(JSON alias_fixture GET "${saved_alias}" alias_fixture) + string(JSON alias_witness GET "${saved_alias}" profile_round_trip) + if(NOT alias_fixture STREQUAL "91" OR NOT alias_witness STREQUAL "731") + message(FATAL_ERROR "Application did not use the borrowed fixture below the ancestor alias") + endif() + set(report_parent "${case_root}") + file(REMOVE "${case_root}/ancestor-alias") + else() + message(STATUS "Profile ancestor-alias probe unavailable on this host: ${alias_result}") + endif() + + file(MAKE_DIRECTORY "${case_root}/linked-marker-profile") + file(CREATE_LINK "${expected_borrowed}/.simplnx-test-profile" "${case_root}/linked-marker-profile/.simplnx-test-profile" SYMBOLIC RESULT marker_link_result) + if(marker_link_result STREQUAL "0") + set(borrowed_env "SIMPLNX_TEST_PROFILE_DIR=${case_root}/linked-marker-profile" "SIMPLNX_TEST_PROFILE_TOKEN=${nonce}") + run_case(linked-marker "${PROFILE_PROBE_EXECUTABLE}" 3 rejected success) + file(REMOVE "${case_root}/linked-marker-profile/.simplnx-test-profile") + else() + message(STATUS "Profile marker-link rejection probe unavailable on this host: ${marker_link_result}") + endif() + +endif() + +# This exact, uniquely created driver tree is caller-owned and all children have exited. +file(READ "${case_root}/.profile-driver" actual_token) +if(NOT actual_token STREQUAL nonce OR IS_SYMLINK "${case_root}") + message(FATAL_ERROR "Driver ownership changed; evidence retained: ${case_root}") +endif() +file(REMOVE_RECURSE "${case_root}") +if(EXISTS "${case_root}") + message(FATAL_ERROR "Cannot remove completed driver fixtures: ${case_root}") +endif() diff --git a/test/simplnx_test_main.cpp b/test/simplnx_test_main.cpp index 48425c5326..6b322c3f17 100644 --- a/test/simplnx_test_main.cpp +++ b/test/simplnx_test_main.cpp @@ -1,4 +1,11 @@ // Catch2 recommends placing these lines by themselves in a translation unit // which will help reduce unnecessary recompilations of the expensive Catch main -#define CATCH_CONFIG_MAIN +#define CATCH_CONFIG_RUNNER #include + +#include "simplnx/UnitTest/OwnedCatchRunner.hpp" + +int main(int argc, char* argv[]) +{ + return nx::core::UnitTest::RunOwnedCatchSession(argc, argv); +}